From d7f31338c2f743fb9a0eadfca7fd39b6f129cd82 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 14 Jan 2025 23:55:26 +0000 Subject: [PATCH 001/105] feat: no global working path with asserts --- LICENSE.md | 2 +- tensilelite/Tensile/BenchmarkProblems.py | 78 ++++++++++++------- tensilelite/Tensile/ClientExecutable.py | 12 ++- tensilelite/Tensile/ClientWriter.py | 30 +++++--- tensilelite/Tensile/Tensile.py | 34 +++++---- tensilelite/Tensile/TensileCreateLibrary.py | 83 +++++++++------------ tensilelite/Tensile/TensileRetuneLibrary.py | 2 +- tensilelite/Tensile/Toolchain/Assembly.py | 41 ++++++---- tensilelite/Tensile/Toolchain/Source.py | 25 ++++--- tensilelite/Tensile/Toolchain/Validators.py | 12 +-- 10 files changed, 180 insertions(+), 139 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 0500ce705e..1c1dfd871d 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (C) 2024 - 2025 Advanced Micro Devices, Inc. +Copyright (C) 2024-2025 Advanced Micro Devices, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 24c89e23d0..514c2cc68b 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -22,12 +22,14 @@ # ################################################################################ +import glob import os import shutil import sys import time from copy import deepcopy +from pathlib import Path from . import ClientExecutable from . import SolutionLibrary @@ -113,13 +115,16 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain): + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, outputPath: Path, buildTmpPath: Path): """Write all the files needed for a given benchmarking step""" + ensurePath(outputPath / "Solutions") + ensurePath(outputPath / "Kernels") - copyStaticFiles() + assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + copyStaticFiles(outputPath) kernels = [] - kernelHelperOjbs = [] + kernelHelperObjs = [] kernelNames = set() kernelHelperNames = set() @@ -136,7 +141,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ for ko in solutionHelperKernels: kname = ko.getKernelName() if kname not in kernelHelperNames: - kernelHelperOjbs.append(ko) + kernelHelperObjs.append(ko) kernelHelperNames.add(kname) kernelSerialNaming = Solution.getSerialNaming(kernels) @@ -146,18 +151,21 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ - globalParameters["WorkingPath"], asmToolchain, srcToolchain, \ - solutions, kernels, kernelHelperOjbs, \ + outputPath, asmToolchain, srcToolchain, \ + solutions, kernels, kernelHelperObjs, \ kernelWriterAssembly, errorTolerant=True ) # ^ this is where solutions is mutated - newLibraryDir = ensurePath(os.path.join(globalParameters["WorkingPath"], 'library')) + newLibraryDirOld = ensurePath(os.path.join(globalParameters["WorkingPath"], 'library')) + newLibraryDir = ensurePath(outputPath / 'library') + assert newLibraryDirOld == str(newLibraryDir), f"newLibraryDirOld={newLibraryDirOld} newLibraryDir={newLibraryDir}" + newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, srcToolchain.compiler) newLibrary.applyNaming(kernelMinNaming) LibraryIO.write(newLibraryFile, Utils.state(newLibrary), globalParameters["LibraryFormat"]) - codeObjectFiles = [os.path.relpath(f, globalParameters["WorkingPath"]) \ + codeObjectFiles = [os.path.relpath(f, outputPath) \ for f in codeObjectFiles] if "TileAwareSelection" in problemType and problemType["TileAwareSelection"]: @@ -194,8 +202,9 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ return codeObjectFiles -def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str +def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, + buildTmpPath: Path, benchmarkProblemsPath: Path ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -209,8 +218,13 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG enableTileSelection = benchmarkProcess.problemType["TileAwareSelection"] groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) + + groupNamePath = benchmarkProblemsPath / groupName pushWorkingPath(groupName) + assert str(groupNamePath) == globalParameters["WorkingPath"], f"Group name working path: {globalParameters['WorkingPath']} and the group name path: {groupNamePath}" + ensurePath(os.path.join(globalParameters["WorkingPath"], "Data")) + ensurePath(groupNamePath / "Data") totalBenchmarkSteps = len(benchmarkProcess) resultsFileBaseFinal = None @@ -242,12 +256,18 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG if benchmarkStep.internalSupportParams: print("# InternalSupportParams: {}".format(benchmarkStep.internalSupportParams)) + shortNamePath = groupNamePath / shortName pushWorkingPath(shortName) - stepBaseDir = globalParameters["WorkingPath"] + assert str(shortNamePath) == globalParameters["WorkingPath"], f"shortNamePath={shortNamePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + + stepBaseDir = shortNamePath # file paths - resultsFileBase = os.path.normpath(os.path.join( \ + resultsFileBaseOld = os.path.normpath(os.path.join( \ globalParameters["WorkingPath"], "../Data", shortName)) + resultsFileBase = os.path.normpath(shortNamePath / ".." / "Data" / shortName) + assert resultsFileBaseOld == resultsFileBase, f"resultsFileBaseOld={resultsFileBaseOld} resultsFileBase={resultsFileBase}" + if benchmarkStep.isFinal(): resultsFileBaseFinal = resultsFileBase resultsFileName = resultsFileBase + ".csv" @@ -255,7 +275,9 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG # check if a solution cache exists and if it matches our solution parameters cachePath = os.path.join(stepBaseDir, "cache.yaml") + sourcePath = ensurePath(shortNamePath / "source") pushWorkingPath("source") + assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" cacheValid = False if useCache and os.path.isfile(cachePath): @@ -310,7 +332,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ - benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain) + benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, sourcePath, buildTmpPath) # ^ this mutates solutions # write cache data @@ -340,12 +362,14 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG ssProblemType = ProblemType(problemTypeConfig) conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) - outFile = os.path.join(globalParameters["WorkingPath"], "ClientParameters.ini") + outFile = os.path.join(sourcePath, "ClientParameters.ini") + + assert sourcePath == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" writeClientConfigIni(True, benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, benchmarkStep.icacheFlushArgs, conProblemType, - globalParameters["WorkingPath"], codeObjectFiles, resultsFileName, + sourcePath, codeObjectFiles, resultsFileName, outFile) # I think the size portion of this yaml could be removed, @@ -359,7 +383,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG if not os.path.exists(resultsFileName) or globalParameters["ForceRedoBenchmarkProblems"]: libraryLogicPath = None forBenchmark = True - returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, srcToolchain.compiler, cCompiler) + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, srcToolchain.compiler, cCompiler, shortNamePath) if returncode: benchmarkTestFails += 1 @@ -379,17 +403,20 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG return (resultsFileBaseFinal, benchmarkTestFails) -def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str): +def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" - ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler) + ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) if config is None: print(f'No config specified in {globalParameters["ConfigPath"]}, built client only') return - dataPath = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) + dataPathOld = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) + benchmarkDataPath = ensurePath(outputPath / globalParameters["BenchmarkDataPath"]) + pushWorkingPath(globalParameters["BenchmarkProblemsPath"]) - ensurePath(dataPath) + assert dataPathOld == str(benchmarkDataPath), f"dataPathOld={dataPathOld} dataPath={benchmarkDataPath}" + totalTestFails = 0 for benchmarkProblemTypeConfig in config: @@ -407,11 +434,11 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # using a suffix to check the csv version (for later addFromCSV()) csvSuffix = "_CSVWinner" if globalParameters["CSVExportWinner"] else "" # results files will be named - newResultsFileName = os.path.join(dataPath, "{}_{:02d}{}.csv" \ + newResultsFileName = os.path.join(benchmarkDataPath, "{}_{:02d}{}.csv" \ .format(str(problemTypeObj), idx, csvSuffix) ) - newSolutionsFileName = os.path.join(dataPath, "{}_{:02d}{}.yaml" \ + newSolutionsFileName = os.path.join(benchmarkDataPath, "{}_{:02d}{}.yaml" \ .format(str(problemTypeObj), idx, csvSuffix) ) - newGranularityFileName = os.path.join(dataPath, "{}_{:02d}{}.gsp" \ + newGranularityFileName = os.path.join(benchmarkDataPath, "{}_{:02d}{}.gsp" \ .format(str(problemTypeObj), idx, csvSuffix) ) # skip if possible @@ -419,8 +446,9 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source or not os.path.exists(newResultsFileName): # benchmark problem size group + benchmarkProblemsPath = ensurePath(outputPath / globalParameters["BenchmarkProblemsPath"]) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler) + benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ @@ -440,7 +468,5 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source print1("# {}_{:02d} already benchmarked; skipping." \ .format(str(problemTypeObj), idx) ) - popWorkingPath() - if globalParameters["ExitOnFails"] and totalTestFails: sys.exit(1) diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index 9cf2c76c3c..35daaca0a7 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -26,6 +26,7 @@ import os import subprocess from typing import Optional +from pathlib import Path from . import Common from .Common import globalParameters @@ -57,8 +58,11 @@ def builtPath(self, path, *paths): def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cCompiler: str): sourcedir = globalParameters["SourcePath"] - if builddir is None: - builddir = os.path.join(globalParameters["OutputPath"], globalParameters["ClientBuildPath"]) + + builddirOld = os.path.join(globalParameters["OutputPath"], globalParameters["ClientBuildPath"]) + assert builddir is not None, "builddir must be provided" + assert builddirOld == str(builddir), f"builddir must be {builddirOld}, not {builddir}" + builddir = Common.ensurePath(builddir) options = {'CMAKE_BUILD_TYPE': globalParameters["CMakeBuildType"], @@ -74,14 +78,14 @@ def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cComp buildEnv = None -def getClientExecutable(cxxCompiler: str, cCompiler: str, builddir=None): +def getClientExecutable(cxxCompiler: str, cCompiler: str, builddir): if "PrebuiltClient" in globalParameters: return globalParameters["PrebuiltClient"] global buildEnv if buildEnv is None: - buildEnv = clientExecutableEnvironment(builddir, cxxCompiler, cCompiler) + buildEnv = clientExecutableEnvironment(builddir / globalParameters["ClientBuildPath"], cxxCompiler, cCompiler) buildEnv.generate() buildEnv.build() diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 4b9f7c1ccc..55d8a72f43 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ # ################################################################################ +from matplotlib.pyplot import step from . import ClientExecutable from . import LibraryIO from .TensileInstructions import getGfxName, DataType @@ -77,13 +78,20 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str): +def main(config, cxxCompiler: str, cCompiler: str, outputPath): libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ globalParameters["LibraryLogicPath"]) - stepBaseDir = pushWorkingPath(globalParameters["LibraryClientPath"]) + assert outputPath == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" + + stepBaseDirOld = pushWorkingPath(globalParameters["LibraryClientPath"]) + stepBaseDir = outputPath / globalParameters["LibraryClientPath"] + assert stepBaseDir == stepBaseDirOld, f"stepBaseDir={stepBaseDir} != SBD={stepBaseDirOld}" pushWorkingPath("source") - copyStaticFiles() + sourcePath = stepBaseDir / "source" + assert sourcePath == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" + + copyStaticFiles(sourcePath) ############################################################################## # Read Logic Files @@ -101,7 +109,7 @@ def main(config, cxxCompiler: str, cCompiler: str): subprocess.run(shlex.split(createLibraryScript), cwd=stepBaseDir) coList = glob(os.path.join(stepBaseDir,"library/*.co")) yamlList = glob(os.path.join(stepBaseDir,"library/*.yaml")) - + clientParametersPaths = [] for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary, _) \ @@ -172,7 +180,7 @@ def main(config, cxxCompiler: str, cCompiler: str): forBenchmark = False enableTileSelection = False - returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, clientParametersPaths) + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, outputPath, clientParametersPaths) popWorkingPath() # LibraryClient @@ -193,12 +201,14 @@ def runNewClient(scriptPath, clientParametersPath, cxxCompiler: str, cCompiler: printWarning("ClientWriter Benchmark Process exited with error: {}".format(e)) -def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, configPaths=None): +def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, outputPath, configPaths=None): # write runScript pushWorkingPath("build") path = globalParameters["WorkingPath"] + buildPath = outputPath / "build" + assert path == str(buildPath), f"path={path} != buildPath={buildPath}" - runScriptName = writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, configPaths) + runScriptName = writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, buildPath, configPaths) with ClientExecutionLock(): process = subprocess.Popen(runScriptName, cwd=path) process.communicate() @@ -262,7 +272,7 @@ def writeBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler): os.chmod(filename, 0o777) return filename -def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, configPaths=None): +def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, buildDir, configPaths=None): if configPaths is None: configPaths = [] configPaths.append(os.path.join(globalParameters["WorkingPath"], "../source/ClientParameters.ini")) @@ -299,7 +309,7 @@ def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cC runScriptFile.write("ERR1=0\n") - clientExe = ClientExecutable.getClientExecutable(cxxCompiler, cCompiler) + clientExe = ClientExecutable.getClientExecutable(cxxCompiler, cCompiler, buildDir) for configFile in configPaths: runScriptFile.write("{} --config-file {} {}\n".format(clientExe, configFile, globalParameters["ClientArgs"])) runScriptFile.write("ERR2=$?\n\n") diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 3b35ba8b06..2d8bef27f4 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -40,6 +40,7 @@ from . import LibraryLogic from . import __version__ from datetime import datetime +from pathlib import Path ############################################################################### @@ -50,20 +51,20 @@ # LibraryLogic.main() to analyse final benchmark data and produce logic/yaml # ClientWriter.main() to create client which calls library based on above yaml ################################################################################ -def executeStepsInConfig(config, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str): +def executeStepsInConfig(config, outputPath: Path, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str): + buildTmpPath = outputPath / "build_tmp" ############################################################################## # Benchmark Problems ############################################################################## if "BenchmarkProblems" in config: - BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler) + BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath) print1("") ############################################################################## # Library Logic ############################################################################## - libraryLogicDataPath = os.path.join(globalParameters["WorkingPath"], \ - globalParameters["LibraryLogicPath"]) + libraryLogicDataPath = os.path.join(outputPath, globalParameters["LibraryLogicPath"]) if "LibraryLogic" in config: if os.path.exists(libraryLogicDataPath): libraryLogicFiles = os.listdir(libraryLogicDataPath) @@ -88,7 +89,7 @@ def executeStepsInConfig(config, asmToolchain: AssemblyToolchain, srcToolchain: libraryClientConfig = config["LibraryClient"] else: libraryClientConfig = {} - ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler) + ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath) print1("") @@ -197,9 +198,9 @@ def Tensile(userArgs): print1("# Tensile v%s" % (__version__)) argParser = argparse.ArgumentParser() - argParser.add_argument("config_file", type=os.path.realpath, nargs="+", + argParser.add_argument("ConfigFile", type=os.path.realpath, nargs="+", help="Benchmark config.yaml file") - argParser.add_argument("output_path", \ + argParser.add_argument("OutputPath", \ help="Path to conduct benchmark and write output files") argParser.add_argument("--version", action="version", \ version="%(prog)s {version}".format(version=__version__)) @@ -212,9 +213,11 @@ def Tensile(userArgs): addCommonArguments(argParser) args = argParser.parse_args(userArgs) - configPaths = args.config_file + configPaths = args.ConfigFile altFormat = args.AlternateFormat useCache = args.useCache + outputPath = Path(ensurePath(os.path.abspath(args.OutputPath))) + print1(f"# OutputPath: {str(outputPath)}") if altFormat and len(configPaths) > 2: printExit("Only 1 or 2 config_files are accepted for the alternate config format: " @@ -274,11 +277,12 @@ def Tensile(userArgs): assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) - asmToolchain = AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) - srcToolchain = SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) + asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) + srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) - globalParameters["OutputPath"] = ensurePath(os.path.abspath(args.output_path)) - globalParameters["WorkingPath"] = globalParameters["OutputPath"] + globalParameters["OutputPath"] = outputPath + globalParameters["WorkingPath"] = outputPath + print("WorkingPath: %s" % globalParameters["WorkingPath"]) overrideParameters = argUpdatedGlobalParameters(args) @@ -295,13 +299,13 @@ def Tensile(userArgs): profiler = cProfile.Profile() profiler.enable() - executeStepsInConfig(config, asmToolchain, srcToolchain, cCompiler) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler) if profiler: profiler.disable() - filename = globalParameters["OutputPath"] + "/tensile.stats" + filename = outputPath / "tensile.stats" profiler.dump_stats(filename) - filename = globalParameters["OutputPath"] + "/tensile.prof" + filename = outputPath / "tensile.prof" profiler.dump_stats(filename) def TensileConfigPath(*args): diff --git a/tensilelite/Tensile/TensileCreateLibrary.py b/tensilelite/Tensile/TensileCreateLibrary.py index 33fe838b86..38076446de 100644 --- a/tensilelite/Tensile/TensileCreateLibrary.py +++ b/tensilelite/Tensile/TensileCreateLibrary.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -36,7 +36,7 @@ from . import LibraryIO from . import Utils from .Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles -from .Toolchain.Source import SourceToolchain, buildSourceCodeObjectFile +from .Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles from .Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults from .TensileInstructions import getGfxName, TensileInstructions from .Common import globalParameters, HR, print1, print2, printExit, ensurePath, \ @@ -137,6 +137,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, for rel in removeResults: results.remove(rel) + def writeAssembly(asmPath: Union[Path, str], result: KernelCodeGenResult): if result.err: printExit(f"Failed to build kernel {result.name} because it has error code {result.err}") @@ -145,8 +146,9 @@ def writeAssembly(asmPath: Union[Path, str], result: KernelCodeGenResult): wfsize = result.wavefrontSize with open(path, "w", encoding="utf-8") as f: f.write(result.src) - del result # result.src is very large so let gc know to clean up asap - + # result.src is very large so let garbage collector know to clean up + del result + return path, isa, wfsize @@ -155,7 +157,7 @@ def writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNE kernelHeaderFilename = os.path.join(os.path.normcase(outputPath), KERNEL_HELPER_FILENAME_H) with open(kernelHeaderFilename, "w", encoding="utf-8") as kernelHeaderFile, \ - open(kernelSourceFilename, "w", encoding="utf-8") as kernelSourceFile: + open(kernelSourceFilename, "w", encoding="utf-8") as kernelSourceFile: kernelSourceFile.write(CHeader) kernelHeaderFile.write(CHeader) kernelSourceFile.write("#include \"Kernels.h\"\n") @@ -163,37 +165,36 @@ def writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNE if globalParameters["RuntimeLanguage"] == "HIP": kernelHeaderFile.write("#include \n") kernelHeaderFile.write("#include \n\n") - kernelHeaderFile.write("#include \"KernelHeader.h\"\n\n") + kernelHeaderFile.write("#include \"KernelHeader.h\"\n\n") HeaderText = "" for ko in kernelHelperObjs: kernelName = ko.getKernelName() - + (err, src) = ko.getSourceFileString() kernelSourceFile.write(src) if err: print("*** warning: invalid kernel#%u" % kernelName) - + HeaderText += ko.getHeaderFileString() - + kernelHeaderFile.write(HeaderText) ################################################################################ # Write Solutions and Kernels for BenchmarkClient or LibraryClient ################################################################################ +@timing def writeSolutionsAndKernels(outputPath, asmToolchain, srcToolchain, solutions, kernels, kernelHelperObjs, \ kernelWriterAssembly, errorTolerant=False, compress=True): + codeObjectFiles = [] - # Push working path into build_tmp folder because there may be more than - # one process running this script. This is to avoid build directory clashing. - # NOTE: file paths must not contain the lower case word 'kernel' or the - # /opt/rocm/bin/extractkernel will fail. - # See buildSourceCodeObjectFile:167 for the call to this binary. - Common.pushWorkingPath('build_tmp') - Common.pushWorkingPath(os.path.basename(outputPath).upper()) - asmPath = ensurePath(os.path.join(globalParameters["WorkingPath"], "assembly")) + outputPath = Path(outputPath) + destLibPath = ensurePath(outputPath / "library") # Destination for code object library files (.co) + buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) + tmpAsmPath = ensurePath(buildTmpPath / "assembly") # Temp path for generated assembly files (.s) + tmpHipCoPath = ensurePath(buildTmpPath / "code_object_tmp") # Temp path for HSA code object files (.hsaco) asmKernels = [k for k in kernels if k['KernelLanguage'] == 'Assembly'] @@ -208,7 +209,7 @@ def writeSolutionsAndKernels(outputPath, asmToolchain, srcToolchain, solutions, duplicates += k.duplicate print2(f"Duplicate: {base}") visited.add(base) - + print1(f"Number of duplicates: {duplicates}") numAsmKernels = len(asmKernels) @@ -220,20 +221,17 @@ def writeSolutionsAndKernels(outputPath, asmToolchain, srcToolchain, solutions, def assemble(ret): p, isa, wavefrontsize = ret asmToolchain.assemble(str(p), str(p.with_suffix(".o")), getGfxName(isa), wavefrontsize) - unaryWriteAssembly = functools.partial(writeAssembly, asmPath) + unaryWriteAssembly = functools.partial(writeAssembly, tmpAsmPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) ret = Common.ParallelMap2(compose(assemble, unaryWriteAssembly), asmResults, "Writing assembly kernels", return_as="list", multiArg=False) - codeObjectFiles += buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, outputPath, compress) + codeObjectFiles += buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, tmpAsmPath, compress) srcKernels = [k for k in kernels if k['KernelLanguage'] != 'Assembly'] if srcKernels: raise ValueError(f"Non-helper HIP source kernels are not supported Tensilelite, found {len(srcKernels)}") writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" - buildSourceCodeObjectFile(srcToolchain, outputPath, srcKernelFile) - - Common.popWorkingPath() # build_tmp - Common.popWorkingPath() # workingDir + buildSourceCodeObjectFiles(srcToolchain, destLibPath, tmpHipCoPath, outputPath, srcKernelFile) return codeObjectFiles, numKernels @@ -241,9 +239,11 @@ def assemble(ret): def writeSolutionsAndKernelsTCL(outputPath, asmToolchain, srcToolchain, kernels, kernelHelperObjs, \ kernelWriterAssembly, compress=True): - Common.pushWorkingPath('build_tmp') - Common.pushWorkingPath(os.path.basename(outputPath).upper()) - asmPath = ensurePath(os.path.join(globalParameters["WorkingPath"], "assembly")) + outputPath = Path(outputPath) + destLibPath = ensurePath(outputPath / "library") # Destination for code object library files (.co) + buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) + tmpAsmPath = ensurePath(buildTmpPath / "assembly") # Temp path for generated assembly files (.s) + tmpHipCoPath = ensurePath(buildTmpPath / "code_object_tmp") # Temp path for HSA code object files (.hsaco) asmKernels = [k for k in kernels if k['KernelLanguage'] == 'Assembly'] @@ -268,17 +268,14 @@ def assemble(ret): p, isa, wavefrontsize = ret asmToolchain.assemble(str(p), str(p.with_suffix(".o")), getGfxName(isa), wavefrontsize) unaryProcessKernelSource = functools.partial(processKernelSource, kernelWriterAssembly, TensileInstructions()) - unaryWriteAssembly = functools.partial(writeAssembly, asmPath) + unaryWriteAssembly = functools.partial(writeAssembly, tmpAsmPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) ret = Common.ParallelMap2(compose(assemble, unaryWriteAssembly, unaryProcessKernelSource), uniqueAsmKernels, "Generating assembly kernels", multiArg=False) - buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, outputPath, compress) + buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, tmpAsmPath, compress) writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" - buildSourceCodeObjectFile(srcToolchain, outputPath, srcKernelFile) - - Common.popWorkingPath() # build_tmp - Common.popWorkingPath() # workingDir + buildSourceCodeObjectFiles(srcToolchain, destLibPath, tmpHipCoPath, outputPath, srcKernelFile) return numKernels @@ -302,9 +299,7 @@ def getSolutionAndKernelWriters(solutions, kernels, assembler): # copy static cpp files and headers ################################################################################ @timing -def copyStaticFiles(outputPath=None): - if outputPath is None: - outputPath = globalParameters["WorkingPath"] +def copyStaticFiles(outputPath): libraryStaticFiles = [ "TensileTypes.h", "tensile_bfloat16.h", @@ -315,9 +310,7 @@ def copyStaticFiles(outputPath=None): "memory_gfx.h" ] for fileName in libraryStaticFiles: - # copy file - shutil.copy( os.path.join(globalParameters["SourcePath"], fileName), \ - outputPath ) + shutil.copy(os.path.join(globalParameters["SourcePath"], fileName), outputPath) return libraryStaticFiles @@ -534,10 +527,9 @@ def splitExtraParameters(par): libraryFormat = args.LibraryFormat useCompression = not args.NoCompress coVersion = args.CodeObjectVersion - + outputPath = Path(ensurePath(os.path.abspath(args.OutputPath))) print2("OutputPath: %s" % outputPath) - ensurePath(outputPath) - outputPath = os.path.abspath(outputPath) + arguments = {} arguments["RuntimeLanguage"] = args.RuntimeLanguage arguments["CodeObjectVersion"] = args.CodeObjectVersion @@ -554,12 +546,7 @@ def splitExtraParameters(par): arguments["LibraryFormat"] = args.LibraryFormat if args.no_enumerate: arguments["AMDGPUArchPath"] = False - arguments["GenerateSourcesAndExit"] = args.GenerateSourcesAndExit - if arguments["GenerateSourcesAndExit"]: - # Generated sources are preserved and go into output dir - arguments["WorkingPath"] = outputPath - arguments["CpuThreads"] = args.CpuThreads arguments["PrintLevel"] = args.PrintLevel arguments["PrintTiming"] = args.PrintTiming @@ -653,7 +640,7 @@ def validLogicFile(p: Path): outputPath ) # write solutions and kernels - numKernels = writeSolutionsAndKernelsTCL(outputPath, asmToolchain, srcToolchain, kernels, + numKernels = writeSolutionsAndKernelsTCL(outputPath, asmToolchain, srcToolchain, kernels, kernelHelperObjs, kernelWriterAssembly, compress=useCompression) archs = [getGfxName(arch) for arch in globalParameters['SupportedISA'] \ diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py index 5c4470f58d..b4464e99ed 100644 --- a/tensilelite/Tensile/TensileRetuneLibrary.py +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -32,7 +32,7 @@ pushWorkingPath, popWorkingPath, restoreDefaultGlobalParameters, HR from .Tensile import addCommonArguments, argUpdatedGlobalParameters from .SolutionStructs import ProblemSizes -from .Utilities.Toolchain import validateToolchain +from .Toolchain.Validators import validateToolchain from . import __version__ import argparse diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 0b44f36dff..7cb02958ff 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -28,14 +28,13 @@ import shlex import shutil import subprocess -import warnings from pathlib import Path -from typing import List, Literal, Union, Tuple +from typing import List, Literal, Union -from .. import Utils from ..TensileInstructions import getGfxName from ..Common import globalParameters, print2, ensurePath + class AssemblyToolchain: def __init__(self, assembler: str, bundler: str, buildIdKind: str, coVersion: Literal[4, 5]): self.assembler = assembler @@ -76,15 +75,15 @@ def assemble(self, srcPath: str, destPath: str, gfx: str, wavefrontSize: int, de """ launcher = shlex.split(os.environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')) args = [ - *launcher, - self.assembler, - "-x", "assembler", - "--target=amdgcn-amd-amdhsa", - f"-mcode-object-version={self.coVersion}", - f"-mcpu={gfx}", + *launcher, + self.assembler, + "-x", "assembler", + "--target=amdgcn-amd-amdhsa", + f"-mcode-object-version={self.coVersion}", + f"-mcpu={gfx}", "-mwavefrontsize64" if wavefrontSize == 64 else "-mno-wavefrontsize64" "-g" if debug else "", - "-c", + "-c", "-o", destPath, srcPath ] @@ -116,7 +115,7 @@ def link(self, srcPaths: List[str], destPath: str): "-Xlinker", f"--build-id={self.buildIdKind}", "-o", destPath, *srcPaths ] - + return self.invoke(args, "Linking assembly object files into code object (*.o -> .co)") def compress(self, srcPath: str, destPath: str, gfx: str): @@ -146,7 +145,7 @@ def compress(self, srcPath: str, destPath: str, gfx: str): def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: numObjFiles = len(objFiles) - + if numObjFiles <= maxObjFiles: return objFiles @@ -167,16 +166,26 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF return newObjFilesOutput -def buildAssemblyCodeObjectFiles(toolchain: AssemblyToolchain, kernels, writerAsm, outputPath, compress: bool=True): - +def buildAssemblyCodeObjectFiles(toolchain: AssemblyToolchain, kernels, writerAsm, destDir, asmDir, compress: bool=True): + """Builds code object files from assembly files + + Args: + toolchain: The AssemblyToolchain object to use. + kernels: A list of dictionaries representing the kernels to build. + writerAsm: The AssemblyWriter object to use. + destDir: The destination directory for the code object files. + asmDir: The directory containing the assembly files. + compress: Whether to compress the code object files. + """ + isAsm = lambda k: k["KernelLanguage"] == "Assembly" extObj = ".o" extCo = ".co" extCoRaw = ".co.raw" - destDir = Path(ensurePath(os.path.join(outputPath, 'library'))) - asmDir = Path(ensurePath(os.path.join(globalParameters["WorkingPath"], "assembly"))) + destDir = Path(ensurePath(destDir)) + asmDir = Path(ensurePath(asmDir)) archKernelMap = collections.defaultdict(list) for k in filter(isAsm, kernels): diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index ab0eba5220..eb74751818 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -82,7 +82,7 @@ def compile(self, srcPath: str, destPath: str, includePath: str, gfxs: List[str] hipFlags = [ "-D__HIP_HCC_COMPAT_MODE__=1", "--offload-device-only", - "-x", "hip", "-O3", + "-x", "hip", "-O3", "-I", includePath, "-Xoffload-linker", f"--build-id={self.buildIdKind}", "-std=c++17", @@ -136,7 +136,7 @@ def unbundle(self, target: str, srcPath: str, destPath: str): ] return self.invoke(args, f"Unbundling source code object file") - + def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Path, str], arch: str) -> Union[Path, None]: """Generates a code object file path using the target, base, and build path. @@ -164,13 +164,14 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa return coPath -def buildSourceCodeObjectFile(toolchain: SourceToolchain, outputPath: Union[Path, str], kernelPath: Union[Path, str]) -> List[str]: +def buildSourceCodeObjectFiles(toolchain: SourceToolchain, destDir: Union[Path, str], tmpObjDir: Union[Path, str], includeDir: Union[Path, str], kernelPath: Union[Path, str]) -> List[str]: """Compiles a HIP source code file into a code object file. Args: - cxxCompiler: The C++ compiler to use. - cxxCompiler: The offload bundler to use. - outputPath: The output directory path where code objects will be placed. + toolchain: The source toolchain. + destDir: The destination directory where HSA code object files are placed. + tmpObjDir: The directory where HIP source object files are created. + includeDir: The include directory path. kernelPath: The path to the kernel source file. Returns: @@ -178,8 +179,8 @@ def buildSourceCodeObjectFile(toolchain: SourceToolchain, outputPath: Union[Path """ start = timer() - buildPath = Path(ensurePath(os.path.join(globalParameters['WorkingPath'], 'code_object_tmp'))) - destPath = Path(ensurePath(os.path.join(outputPath, 'library'))) + tmpObjDir = Path(ensurePath(tmpObjDir)) + destDir = Path(ensurePath(destDir)) kernelPath = Path(kernelPath) if "CmakeCxxCompiler" in globalParameters and globalParameters["CmakeCxxCompiler"] is not None: @@ -191,18 +192,18 @@ def buildSourceCodeObjectFile(toolchain: SourceToolchain, outputPath: Union[Path _, cmdlineArchs = splitArchs() - objPath = str(buildPath / objFilename) - toolchain.compile(str(kernelPath), objPath, str(outputPath), cmdlineArchs) + objPath = str(tmpObjDir / objFilename) + toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) for target in toolchain.targets(objPath): match = re.search("gfx.*$", target) if match: arch = re.sub(":", "-", match.group()) - coPathRaw = _computeSourceCodeObjectFilename(target, kernelPath.stem, buildPath, arch) + coPathRaw = _computeSourceCodeObjectFilename(target, kernelPath.stem, tmpObjDir, arch) if not coPathRaw: continue toolchain.unbundle(target, objPath, str(coPathRaw)) - coPath = str(destPath / coPathRaw.stem) + coPath = str(destDir / coPathRaw.stem) coPathsRaw.append(coPathRaw) coPaths.append(coPath) diff --git a/tensilelite/Tensile/Toolchain/Validators.py b/tensilelite/Tensile/Toolchain/Validators.py index 86c3ff074b..a67c9bb7e1 100644 --- a/tensilelite/Tensile/Toolchain/Validators.py +++ b/tensilelite/Tensile/Toolchain/Validators.py @@ -34,9 +34,9 @@ if os.name == "nt": def _windowsLatestRocmBin(path: Union[Path, str]) -> Path: """Get the path to the latest ROCm bin directory, on Windows. - + This function assumes that ROCm versions are differentiated with the form ``X.Y``. - + Args: path: The path to the ROCm root directory, typically ``C:/Program Files/AMD/ROCm``. @@ -147,7 +147,7 @@ def _validateExecutable(file: str, searchPaths: List[Path]) -> str: if _exeExists(Path(file)): return file for path in searchPaths: - path /= file + path /= file if _exeExists(path): return str(path) raise FileNotFoundError(f"`{file}` either not found or not executable in any search path: {':'.join(map(str, searchPaths))}") @@ -157,10 +157,10 @@ def validateToolchain(*args: str): Args: args: List of executable toolchain components to validate. - + Returns: List of validated executables with absolute paths. - + Raises: ValueError: If no toolchain components are provided. FileNotFoundError: If a toolchain component is not found in the PATH. @@ -174,7 +174,7 @@ def validateToolchain(*args: str): ] + [Path(p) for p in os.environ["PATH"].split(os.pathsep)] out = (_validateExecutable(x, searchPaths) for x in args) - return next(out) if len(args) == 1 else tuple(out) + return next(out) if len(args) == 1 else tuple(out) def getVersion(executable: str, versionFlag: str="--version", regex: str=r"version\s+([\d.]+)") -> str: From a1375c71dfe1c7d4c603eb202efc821c8bca7703 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:17:56 +0000 Subject: [PATCH 002/105] fix: bad import statement --- tensilelite/Tensile/ClientWriter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 55d8a72f43..c1835cc180 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -22,7 +22,6 @@ # ################################################################################ -from matplotlib.pyplot import step from . import ClientExecutable from . import LibraryIO from .TensileInstructions import getGfxName, DataType From 07cd30cbc2f1444b56a85d1566c22e8c4476b0c1 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 15 Jan 2025 20:48:42 +0000 Subject: [PATCH 003/105] fix: different build_tmp dir in cmake than tensile --- tensilelite/Tensile/cmake/TensileConfig.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/cmake/TensileConfig.cmake b/tensilelite/Tensile/cmake/TensileConfig.cmake index 1455e023d0..8cd9970984 100644 --- a/tensilelite/Tensile/cmake/TensileConfig.cmake +++ b/tensilelite/Tensile/cmake/TensileConfig.cmake @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -88,7 +88,7 @@ function(TensileCreateLibraryFiles ASAN_BUILD KEEP_BUILD_TMP NO_COMPRESS - EXPERIMENTAL + EXPERIMENTAL ENABLE_MAKRER ) @@ -273,7 +273,7 @@ endfunction() function(TensileCreateExtOpLibraries OutputFolder ArchStr) string(REGEX MATCHALL "gfx[a-z0-9]+" Archs "${ArchStr}") list(REMOVE_DUPLICATES Archs) - set(build_tmp_dir ${CMAKE_CURRENT_BINARY_DIR}/build_tmp/ops) + set(build_tmp_dir ${OutputFolder}/../build_tmp/ops) set(Tensile_PACKAGE_DIR ${Tensile_SOURCE_DIR}/../) set(cwd "${Tensile_PACKAGE_DIR}/Ops") set(script "${cwd}/gen_assembly.sh") From b5b0f4957ef2df562ef25f9580f038f602ebbe36 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 17 Jan 2025 19:03:17 +0000 Subject: [PATCH 004/105] fix: incorrect pathing in Tensile benchmarking --- tensilelite/Tensile/BenchmarkProblems.py | 74 +- tensilelite/Tensile/ClientWriter.py | 1097 ++----------------- tensilelite/Tensile/Common.py | 18 +- tensilelite/Tensile/LibraryLogic.py | 57 +- tensilelite/Tensile/Tensile.py | 38 +- tensilelite/Tensile/TensileRetuneLibrary.py | 7 +- 6 files changed, 231 insertions(+), 1060 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 514c2cc68b..54bb64830d 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -115,13 +115,16 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, outputPath: Path, buildTmpPath: Path): + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, buildTmpPath: Path): """Write all the files needed for a given benchmarking step""" - ensurePath(outputPath / "Solutions") - ensurePath(outputPath / "Kernels") + ensurePath(sourcePath / "Solutions") + ensurePath(sourcePath / "Kernels") - assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - copyStaticFiles(outputPath) + # deleteme + assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"outputPath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + # deleteme + + copyStaticFiles(sourcePath) kernels = [] kernelHelperObjs = [] @@ -151,21 +154,24 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ - outputPath, asmToolchain, srcToolchain, \ + sourcePath, asmToolchain, srcToolchain, \ solutions, kernels, kernelHelperObjs, \ kernelWriterAssembly, errorTolerant=True ) # ^ this is where solutions is mutated + newLibraryDir = ensurePath(sourcePath / 'library') + + # deleteme newLibraryDirOld = ensurePath(os.path.join(globalParameters["WorkingPath"], 'library')) - newLibraryDir = ensurePath(outputPath / 'library') assert newLibraryDirOld == str(newLibraryDir), f"newLibraryDirOld={newLibraryDirOld} newLibraryDir={newLibraryDir}" + # deleteme newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, srcToolchain.compiler) newLibrary.applyNaming(kernelMinNaming) LibraryIO.write(newLibraryFile, Utils.state(newLibrary), globalParameters["LibraryFormat"]) - codeObjectFiles = [os.path.relpath(f, outputPath) \ + codeObjectFiles = [os.path.relpath(f, sourcePath) \ for f in codeObjectFiles] if "TileAwareSelection" in problemType and problemType["TileAwareSelection"]: @@ -190,11 +196,13 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ idealSize = {"Exact": [idealM, idealN, idealK]} idealSizes.append(idealSize) idealProblemSizes = ProblemSizes(problemType, idealSizes) - writeClientConfig(True, solutions, idealProblemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, True) + writeClientConfig(True, solutions, idealProblemSizes, biasTypeArgs, \ + factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ + newLibrary, codeObjectFiles, True) else: - writeClientConfig(True, solutions, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, False) + writeClientConfig(True, solutions, problemSizes, biasTypeArgs, \ + factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ + newLibrary, codeObjectFiles, False) if len(solutions) == 0: printExit("write solutions and kernels results 0 valid soultion.") @@ -220,10 +228,14 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) groupNamePath = benchmarkProblemsPath / groupName + + # deleteme pushWorkingPath(groupName) assert str(groupNamePath) == globalParameters["WorkingPath"], f"Group name working path: {globalParameters['WorkingPath']} and the group name path: {groupNamePath}" ensurePath(os.path.join(globalParameters["WorkingPath"], "Data")) + # deleteme + ensurePath(groupNamePath / "Data") totalBenchmarkSteps = len(benchmarkProcess) @@ -257,16 +269,21 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG print("# InternalSupportParams: {}".format(benchmarkStep.internalSupportParams)) shortNamePath = groupNamePath / shortName + # deleteme pushWorkingPath(shortName) assert str(shortNamePath) == globalParameters["WorkingPath"], f"shortNamePath={shortNamePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + # deleteme stepBaseDir = shortNamePath # file paths + resultsFileBase = os.path.normpath(shortNamePath / ".." / "Data" / shortName) + + # deleteme resultsFileBaseOld = os.path.normpath(os.path.join( \ globalParameters["WorkingPath"], "../Data", shortName)) - resultsFileBase = os.path.normpath(shortNamePath / ".." / "Data" / shortName) assert resultsFileBaseOld == resultsFileBase, f"resultsFileBaseOld={resultsFileBaseOld} resultsFileBase={resultsFileBase}" + # deleteme if benchmarkStep.isFinal(): resultsFileBaseFinal = resultsFileBase @@ -276,8 +293,11 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG # check if a solution cache exists and if it matches our solution parameters cachePath = os.path.join(stepBaseDir, "cache.yaml") sourcePath = ensurePath(shortNamePath / "source") + + # deleteme pushWorkingPath("source") assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + # deleteme cacheValid = False if useCache and os.path.isfile(cachePath): @@ -329,10 +349,11 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG # write benchmarkFiles prevCount = len(solutions) - codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ + codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ - benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, sourcePath, buildTmpPath) + benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ + sourcePath, buildTmpPath) # ^ this mutates solutions # write cache data @@ -364,12 +385,15 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) outFile = os.path.join(sourcePath, "ClientParameters.ini") - assert sourcePath == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + # deleteme + assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"stepBaseDir={stepBaseDir / 'source'} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" + # deleteme writeClientConfigIni(True, benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, benchmarkStep.icacheFlushArgs, conProblemType, - sourcePath, codeObjectFiles, resultsFileName, + stepBaseDir, codeObjectFiles, resultsFileName, outFile) # I think the size portion of this yaml could be removed, @@ -377,7 +401,9 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG LibraryIO.writeSolutions(solutionsFileName, benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, benchmarkStep.activationArgs, solutions, cacheValid) + # deleteme popWorkingPath() # source + # deleteme # run benchmarking client if not os.path.exists(resultsFileName) or globalParameters["ForceRedoBenchmarkProblems"]: @@ -393,13 +419,17 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG print1("# Already benchmarked; skipping.") # End Iteration + # deleteme popWorkingPath() # stepName + # deleteme currentTime = time.time() elapsedTime = currentTime - startTime print1("{}\n# {}\n# {}: End - {:.3f}s\n{}\n" \ .format(HR, groupName, shortName, elapsedTime, HR)) - + # deleteme popWorkingPath() # ProblemType + # deleteme + return (resultsFileBaseFinal, benchmarkTestFails) @@ -411,11 +441,13 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source print(f'No config specified in {globalParameters["ConfigPath"]}, built client only') return - dataPathOld = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) benchmarkDataPath = ensurePath(outputPath / globalParameters["BenchmarkDataPath"]) + # deleteme + dataPathOld = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) pushWorkingPath(globalParameters["BenchmarkProblemsPath"]) assert dataPathOld == str(benchmarkDataPath), f"dataPathOld={dataPathOld} dataPath={benchmarkDataPath}" + # deleteme totalTestFails = 0 @@ -468,5 +500,9 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source print1("# {}_{:02d} already benchmarked; skipping." \ .format(str(problemTypeObj), idx) ) + # deleteme + popWorkingPath() # BenchmarkProblemsPath + # deleteme + if globalParameters["ExitOnFails"] and totalTestFails: sys.exit(1) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index c1835cc180..a41d664624 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -22,6 +22,7 @@ # ################################################################################ +from pathlib import Path from . import ClientExecutable from . import LibraryIO from .TensileInstructions import getGfxName, DataType @@ -77,18 +78,26 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str, outputPath): +def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): + + libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] + # deleteme libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ globalParameters["LibraryLogicPath"]) - assert outputPath == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" + assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" + # deleteme + clientLibraryPath = outputPath / globalParameters["LibraryClientPath"] + # deleteme stepBaseDirOld = pushWorkingPath(globalParameters["LibraryClientPath"]) - stepBaseDir = outputPath / globalParameters["LibraryClientPath"] - assert stepBaseDir == stepBaseDirOld, f"stepBaseDir={stepBaseDir} != SBD={stepBaseDirOld}" + assert str(clientLibraryPath) == stepBaseDirOld, f"stepBaseDir={clientLibraryPath} != SBD={stepBaseDirOld}" + # deleteme + sourcePath = clientLibraryPath / "source" + # deleteme pushWorkingPath("source") - sourcePath = stepBaseDir / "source" - assert sourcePath == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" + assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" + # deleteme copyStaticFiles(sourcePath) @@ -104,10 +113,10 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath): functionNames = [] enableHalf = False - createLibraryScript = getBuildClientLibraryScript(stepBaseDir, libraryLogicPath, cxxCompiler) - subprocess.run(shlex.split(createLibraryScript), cwd=stepBaseDir) - coList = glob(os.path.join(stepBaseDir,"library/*.co")) - yamlList = glob(os.path.join(stepBaseDir,"library/*.yaml")) + createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler) + subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) + coList = glob(os.path.join(clientLibraryPath, "library/*.co")) + yamlList = glob(os.path.join(clientLibraryPath, "library/*.yaml")) clientParametersPaths = [] for logicFileName in logicFiles: @@ -145,6 +154,12 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath): isForAll = True if problemType["ActivationType"] in ['all', 'hipblaslt_all'] else False activationArgs = ActivationArgs(problemType, activationEnums) if isForAll else "" factorDimArgs = FactorDimArgs(problemType, factorDimEnums) + + # deleteme + assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" + print1(f"sourcePath={sourcePath} == WP={globalParameters['WorkingPath']}") + # deleteme + clientParametersPaths.append(writeClientConfig( forBenchmark=False, solutions=None, @@ -154,7 +169,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath): activationArgs=activationArgs, icacheFlushArgs=icacheFlushArgs, stepName=str(ProblemType(problemType)), - stepBaseDir=globalParameters["WorkingPath"], + stepBaseDir=str(sourcePath), newLibrary=newLibrary, configBase="ClientParameters_%s"%str(ProblemType(problemType)), codeObjectFiles=coList, @@ -167,21 +182,32 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath): ############################################################################## forBenchmark = False problemSizes = None + + # deleteme popWorkingPath() # source + # deleteme ############################################################################## # Run Build Script ############################################################################## # if redo=true, clobber the build directory + + # deleteme + assert str(clientLibraryPath) == globalParameters["WorkingPath"], f"clientLibraryPath={clientLibraryPath} != WP={globalParameters['WorkingPath']}" + # deleteme + if globalParameters["ForceRedoLibraryClient"]: - shutil.rmtree(os.path.join(globalParameters["WorkingPath"], "build"), \ + shutil.rmtree(os.path.join(clientLibraryPath, "build"), \ ignore_errors=True) forBenchmark = False enableTileSelection = False returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, outputPath, clientParametersPaths) + # deleteme popWorkingPath() # LibraryClient + assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" + # deleteme return returncode @@ -202,19 +228,25 @@ def runNewClient(scriptPath, clientParametersPath, cxxCompiler: str, cCompiler: def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, outputPath, configPaths=None): # write runScript - pushWorkingPath("build") - path = globalParameters["WorkingPath"] + buildPath = outputPath / "build" - assert path == str(buildPath), f"path={path} != buildPath={buildPath}" + # deleteme + pushWorkingPath("build") + wkpath = globalParameters["WorkingPath"] + assert wkpath == str(buildPath), f"path={wkpath} != buildPath={buildPath}" + # deleteme - runScriptName = writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, buildPath, configPaths) + runScriptName = writeRunScript(buildPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, buildPath, configPaths) with ClientExecutionLock(): - process = subprocess.Popen(runScriptName, cwd=path) + process = subprocess.Popen(runScriptName, cwd=buildPath) process.communicate() if process.returncode: printWarning("ClientWriter Benchmark Process exited with code %u" % process.returncode) + + # deleteme popWorkingPath() # build + # deleteme return process.returncode @@ -274,12 +306,17 @@ def writeBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler): def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, buildDir, configPaths=None): if configPaths is None: configPaths = [] - configPaths.append(os.path.join(globalParameters["WorkingPath"], "../source/ClientParameters.ini")) + + # deleteme + assert str(buildDir) == globalParameters["WorkingPath"], f"buildDir={buildDir} != WP={globalParameters['WorkingPath']}" + # deleteme + + configPaths.append(os.path.join(buildDir, "../source/ClientParameters.ini")) if enableTileSelection is True: - configPaths.append(os.path.join(globalParameters["WorkingPath"], "../source/ClientParameters_Granularity.ini")) + configPaths.append(os.path.join(buildDir, "../source/ClientParameters_Granularity.ini")) # create run.bat or run.sh which builds and runs - runScriptName = os.path.join(path, \ + runScriptName = os.path.join(buildDir, \ "run.%s" % ("bat" if os.name == "nt" else "sh") ) runScriptFile = open(runScriptName, "w") if os.name != "nt": @@ -529,6 +566,8 @@ def pruneModeName(mode): def writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, libraryFile=None): + assert os.path.exists(sourceDir), f"sourceDir={sourceDir} does not exist" + with open(parametersFilePath, "w") as f: def param(key, value): f.write("{}={}\n".format(key, value)) @@ -668,12 +707,33 @@ def param(key, value): param("rotating-buffer-mode", globalParameters["RotatingMode"]) -def writeClientConfig(forBenchmark, solutions, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, newLibrary, codeObjectFiles, tileAwareSelection, configBase = "ClientParameters", libraryFile = None): +def writeClientConfig( + forBenchmark, + solutions, + problemSizes, + biasTypeArgs, + factorDimArgs, + activationArgs, + icacheFlushArgs, + stepName, + stepBaseDir, + newLibrary, + codeObjectFiles, + tileAwareSelection, + configBase = "ClientParameters", + libraryFile = None + ): + + # deleteme + assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"stepBaseDir={stepBaseDir} != WP={globalParameters['WorkingPath']}" + # deleteme + + sourceDir = os.path.join(stepBaseDir, "source") if tileAwareSelection: - filename = os.path.join(globalParameters["WorkingPath"], "%s_Granularity.ini"%configBase) + filename = os.path.join(sourceDir, "%s_Granularity.ini"%configBase) else: - filename = os.path.join(globalParameters["WorkingPath"], "%s.ini"%configBase) + filename = os.path.join(sourceDir, "%s.ini"%configBase) if len(newLibrary.solutions)==0: raise RuntimeError ("No valid solutions found") @@ -685,7 +745,6 @@ def writeClientConfig(forBenchmark, solutions, problemSizes, biasTypeArgs, facto resultsFileName = os.path.join(stepBaseDir, "../Data", stepName+".csv") newSolution = next(iter(newLibrary.solutions.values())) - sourceDir = os.path.join(stepBaseDir, "source") writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, libraryFile) return filename @@ -708,993 +767,3 @@ def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataF problemType = ContractionsProblemType.FromOriginalState(problemTypeDict) writeClientConfigIni(True, problemSizes, "", "", "", "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile) - - -################################################################################ -# Write Generated Benchmark Parameters -################################################################################ -def writeClientParameters(forBenchmark, solutions, problemSizes, stepName, \ - functionList, stepBaseDir, solutionSummationSizes, solutionWriter = None): - h = "" - - ############################################################################## - # Min Naming - ############################################################################## - """ - if forBenchmark: - kernels = [] - for solution in solutions: - solutionKernels = solution.getKernels() - for kernel in solutionKernels: - if kernel not in kernels: - kernels.append(kernel) - - solutionSerialNaming = Solution.getSerialNaming(solutions) - kernelSerialNaming = Solution.getSerialNaming(kernels) - solutionMinNaming = Solution.getMinNaming(solutions) - kernelMinNaming = Solution.getMinNaming(kernels) - """ - - if forBenchmark: - h += "#include \"Solutions.h\"\n" - h += "#include \"ReferenceCPU.h\"\n" - h += "\n" - else: - h += "#include \"Solutions.h\"\n" - h += "#include \"Tensile.h\"\n" - - - h += "typedef enum {\n" - h += " enum_float,\n" - h += " enum_double,\n" - h += " enum_TensileComplexFloat,\n" - h += " enum_TensileComplexDouble\n" - h += "#ifdef Tensile_ENABLE_HALF\n" - h += " ,enum_TensileHalf\n" - h += "#endif\n" - h += " ,enum_TensileInt8x4\n" - h += " ,enum_TensileInt32\n" - h += " ,enum_tensile_bfloat16\n" - h += "} DataTypeEnum;\n" - h += "\n" - - h += "// Debug Params\n" - h += "const unsigned printTensorA=%x;\n" % int(globalParameters["PrintTensorA"]) - h += "const unsigned printTensorB=%x;\n" % int(globalParameters["PrintTensorB"]) - h += "const unsigned printTensorC=%x;\n" % int(globalParameters["PrintTensorC"]) - h += "const unsigned printTensorD=%x;\n" % int(globalParameters["PrintTensorD"]) - - h += "const bool printWinnersOnly=%s;\n" % toCppBool(globalParameters["PrintWinnersOnly"]) - h += "\n" - - h += "const char indexChars[%u] = \"%s" \ - % (len(globalParameters["IndexChars"])+1, \ - globalParameters["IndexChars"][0]) - for i in range(1, len(globalParameters["IndexChars"])): - h += globalParameters["IndexChars"][i] - h += "\";\n" - - h += "unsigned int functionIdx;\n" - h += "unsigned int dataTypeIdx;\n" - h += "unsigned int problemTypeIdx;\n" - h += "\n" - - ############################################################################## - # Problem Types - ############################################################################## - #dataTypes = [] - #problemTypes = [] - #functionSerialToDataTypeAndIdx = [] - dataTypes = [] - problemTypes = [] - destDataTypes = {} - computeDataTypes = {} - problemTypesForDataType = {} # for data type - schedulesForProblemType = {} # for problem type - functionInfo = [] # dataTypeIdx, problemTypeIdx, idxWithinDataType, idxWithinProblemType - #tileSelection = False - - if forBenchmark: - problemType = solutions[0]["ProblemType"] - dataType = problemType["DataType"] - #tileSelection = problemType["TileAwareSelection"] - - destDataType = problemType["DestDataType"] - destDataTypes[dataType] = destDataType - - computeDataType = problemType["ComputeDataType"] - computeDataTypes[dataType] = computeDataType - - dataTypes.append(dataType) - - problemTypes.append(problemType) - problemTypesForDataType[dataType] = [problemType] - schedulesForProblemType[problemType] = solutions - numProblemTypes = 1 - for solution in solutions: - functionInfo.append([ 0, 0, 0, 0, 0, 0 ]) - else: - for functionIdx in range(0, len(functionList)): - function = functionList[functionIdx] - scheduleName = function[0] - problemType = function[1] - dataType = problemType["DataType"] - destDataType = problemType["DestDataType"] - computeDataType = problemType["ComputeDataType"] - if dataType not in dataTypes: - dataTypes.append(dataType) - destDataTypes[dataType] = destDataType - computeDataTypes[dataType] = computeDataType - problemTypesForDataType[dataType] = [] - if problemType not in problemTypesForDataType[dataType]: - problemTypesForDataType[dataType].append(problemType) - schedulesForProblemType[problemType] = [] - schedulesForProblemType[problemType].append(scheduleName) - - # sort - dataTypes = sorted(dataTypes) - for dataType in dataTypes: - problemTypesForDataType[dataType] = \ - sorted(problemTypesForDataType[dataType],key=str) - for problemType in problemTypesForDataType[dataType]: - schedulesForProblemType[problemType] = \ - sorted(schedulesForProblemType[problemType],key=str) - - # assign info - functionIdxSerial = 0 - problemTypeIdxSerial = 0 - for dataTypeIdxSerial in range(0, len(dataTypes)): - dataType = dataTypes[dataTypeIdxSerial] - functionIdxForDataType = 0 - for problemTypeIdxForDataType in range(0, \ - len(problemTypesForDataType[dataType])): - problemType = \ - problemTypesForDataType[dataType][problemTypeIdxForDataType] - problemTypes.append(problemType) - functionIdxForProblemType = 0 - for functionIdxForProblemType in range(0, \ - len(schedulesForProblemType[problemType])): - functionInfo.append([ \ - dataTypeIdxSerial, \ - problemTypeIdxForDataType, \ - problemTypeIdxSerial, \ - functionIdxSerial,\ - functionIdxForDataType,\ - functionIdxForProblemType, \ - ]) - functionIdxForProblemType += 1 - functionIdxForDataType += 1 - functionIdxSerial += 1 - problemTypeIdxSerial += 1 - numProblemTypes = problemTypeIdxSerial - numFunctions = functionIdxSerial - h += "const unsigned int numFunctions = %u;\n" % numFunctions - - ############################################################################## - # Data Types - ############################################################################## - h += "/* data types */\n" - numDataTypes = len(dataTypes) - h += "const unsigned int numDataTypes = %u;\n" % numDataTypes - h += "const DataTypeEnum dataTypeEnums[numDataTypes] = { enum_%s" \ - % dataTypes[0].toCpp() - for dataTypeIdx in range(1, numDataTypes): - h += ", enum_%s" % dataTypes[dataTypeIdx].toCpp() - h += " };\n" - # bytes per elements - h += "const unsigned int bytesPerElement[numDataTypes] = { %u" \ - % (dataTypes[0].numBytes()) - for dataTypeIdx in range(1, numDataTypes): - dataType = dataTypes[dataTypeIdx] - h += ", %u" % dataType.numBytes() - h += " };\n" - # flops per mac - if dataTypes[0].isInt8x4(): - h += "const unsigned int numFlopsPerMac[numDataTypes] = { %u" % (8 if dataTypes[0].isReal() else 32) - else: - h += "const unsigned int numFlopsPerMac[numDataTypes] = { %u" % (2 if dataTypes[0].isReal() else 8) - for dataTypeIdx in range(1, numDataTypes): - dataType = dataTypes[dataTypeIdx] - h += ", %u" % (2 if dataType.isReal() else 8) - h += " };\n" - for dataTypeIdx in range(0, numDataTypes): - h += "#define Tensile_DATA_TYPE_%s\n" \ - % dataTypes[dataTypeIdx].toCpp().upper() - - ############################################################################## - # Problem Types - ############################################################################## - h += "/* problem types */\n" - h += "const unsigned int numProblemTypes = %u;\n" % numProblemTypes - # Num C Indices - h += "const unsigned int numIndicesC[numProblemTypes] = { %u" \ - % problemTypes[0]["NumIndicesC"] - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - h += ", %u" % problemType["NumIndicesC"] - h += " };\n" - - # Num AB Indices - maxNumIndicesA = len(problemTypes[0]["IndexAssignmentsA"]) - maxNumIndicesB = len(problemTypes[0]["IndexAssignmentsB"]) - h += "const unsigned int numIndicesA[numProblemTypes] = { %u" \ - % len(problemTypes[0]["IndexAssignmentsA"]) - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - numIndicesA = len(problemType["IndexAssignmentsA"]) - h += ", %u" % numIndicesA - maxNumIndicesA = max(numIndicesA, maxNumIndicesA) - h += " };\n" - h += "const unsigned int maxNumIndicesA = %u;\n" % maxNumIndicesA - - h += "const unsigned int numIndicesB[numProblemTypes] = { %u" \ - % len(problemTypes[0]["IndexAssignmentsB"]) - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - numIndicesB = len(problemType["IndexAssignmentsB"]) - h += ", %u" % numIndicesB - maxNumIndicesB = max(numIndicesB, maxNumIndicesB) - h += " };\n" - h += "const unsigned int maxNumIndicesB = %u;\n" % maxNumIndicesB - - # Index Assignments A - h += "const unsigned int indexAssignmentsA[numProblemTypes][maxNumIndicesA] = {\n" - for problemTypeIdx in range(0, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - indices = problemType["IndexAssignmentsA"] - h += " { %u" % indices[0] - for i in range(1, maxNumIndicesA): - if i < len(indices): - h += ", %u" % indices[i] - else: - h += ", static_cast(-1)" - if problemTypeIdx < numProblemTypes-1: - h += " },\n" - else: - h += " }\n" - h += "};\n" - # Index Assignments B - h += "const unsigned int indexAssignmentsB[numProblemTypes][maxNumIndicesB] = {\n" - for problemTypeIdx in range(0, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - indices = problemType["IndexAssignmentsB"] - h += " { %u" % indices[0] - for i in range(1, maxNumIndicesB): - if i < len(indices): - h += ", %u" % indices[i] - else: - h += ", static_cast(-1)" - if problemTypeIdx < numProblemTypes-1: - h += " },\n" - else: - h += " }\n" - h += "};\n" - # Index Assignments LD - h += "const unsigned int numIndicesLD = %u;\n" % problemType["NumIndicesLD"] - h += "const unsigned int indexAssignmentsLD[numIndicesLD] = {" - if problemType["NumIndicesLD"] > 0: - h += " %u" % problemType["IndexAssignmentsLD"][0] - for ldIdx in range(1, len(problemType["IndexAssignmentsLD"])): - h += ", %u" % problemType["IndexAssignmentsLD"][ldIdx] - h += "};\n" - # beta - h += "bool useBeta[numProblemTypes] = { %s" \ - % ("true" if problemTypes[0]["UseBeta"] else "false") - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - h += ", %s" % ("true" if problemType["UseBeta"] else "false") - h += " };\n" - # Complex Conjugates - h += "const bool complexConjugateA[numProblemTypes] = { %s" \ - % ("true" if problemTypes[0]["ComplexConjugateA"] else "false" ) - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - h += ", %s" % ("true" if problemTypes[0]["ComplexConjugateA"] else "false" ) - h += " };\n" - h += "const bool complexConjugateB[numProblemTypes] = { %s" \ - % ("true" if problemTypes[0]["ComplexConjugateB"] else "false" ) - for problemTypeIdx in range(1, numProblemTypes): - problemType = problemTypes[problemTypeIdx] - h += ", %s" % ("true" if problemTypes[0]["ComplexConjugateB"] else "false" ) - h += " };\n" - h += "\n" - - if not forBenchmark: - h += "// dataTypeIdxSerial, problemTypeIdxForDataType, problemTypeIdxSerial, functionIdxSerial, functionIdxForDataType, functionIdxForProblemType\n" - first = True - h += "const unsigned int functionInfo[numFunctions][6] = {\n" - for info in functionInfo: - h += "%s{ %u, %u, %u, %u, %u, %u }" % (" " if first else ",\n ", \ - info[0], info[1], info[2], info[3], info[4], info[5] ) - first = False - h += " };\n" - - - ############################################################################## - # Problem Sizes - ############################################################################## - maxNumIndices = problemTypes[0]["TotalIndices"] - if not forBenchmark: - for problemType in problemTypes: - maxNumIndices = max(problemType["TotalIndices"], maxNumIndices) - h += "const unsigned int maxNumIndices = %u;\n" % maxNumIndices - h += "const unsigned int totalIndices[numProblemTypes] = { %u" \ - % problemTypes[0]["TotalIndices"] - for problemTypeIdx in range(1, numProblemTypes): - h += ", %u" % problemTypes[problemTypeIdx]["TotalIndices"] - h += " };\n" - if forBenchmark: - h += "const unsigned int numProblems = %u;\n" \ - % problemSizes.totalProblemSizes - h += "const unsigned int problemSizes[numProblems][%u] = {\n" \ - % (problemTypes[0]["TotalIndices"] + problemType["NumIndicesLD"]) - for i in range(problemSizes.totalProblemSizes): - #assert problemSizes.problems[i].stridesA == None # new stride functionality only supported on new client, not here - problemSize = problemSizes.problems[i].sizes - line = " {%5u" %problemSize[0] - for j in range(1, problemTypes[0]["TotalIndices"] + problemType["NumIndicesLD"]): - line += ",%5u" % problemSize[j] - line += " }" - h += line - if i < problemSizes.totalProblemSizes-1: - h += "," - else: - h += "" - h += "};\n" - h += "const unsigned int minStrides[%u] = {" \ - % problemTypes[0]["TotalIndices"] - for i in range(0, len(problemSizes.minStrides)): - if (i!=0): - h += ", " - h += str(problemSizes.minStrides[i]) - h += "};\n" - else: - h += "unsigned int userSizes[maxNumIndices];\n" - h += "unsigned int minStrides[%u] = {" \ - % maxNumIndices - for i in range(0, maxNumIndices): - if (i!=0): - h += ", " - h += str(0); # always use 0 for minStrides in benchmark mode - h += "};\n" - - if forBenchmark: - h += "/* problem sizes */\n" - """ - h += "const bool indexIsSized[maxNumIndices] = {" - for i in range(0, problemSizes.totalIndices): - h += " %s" % ("true" if problemSizes.indexIsSized[i] else "false") - if i < problemSizes.totalIndices-1: - h += "," - h += " };\n" - - h += "const unsigned int numIndicesSized = %u;\n" \ - % len(problemSizes.indicesSized) - h += "const unsigned int indicesSized[numIndicesSized][4] = {\n" - h += "// { min, stride, stride_incr, max }\n" - for i in range(0, len(problemSizes.indicesSized)): - r = problemSizes.indicesSized[i] - h += " { %u, %u, %u, %u }" % (r[0], r[1], r[2], r[3]) - if i < len(problemSizes.indicesSized)-1: - h += "," - h += "\n" - h += " };\n" - - numIndicesMapped = len(problemSizes.indicesMapped) - h += "const unsigned int numIndicesMapped = %u;\n" % numIndicesMapped - if numIndicesMapped > 0: - h += "#define Tensile_INDICES_MAPPED 1\n" - h += "const unsigned int indicesMapped[numIndicesMapped] = {" - for i in range(0, numIndicesMapped): - h += " %u" % problemSizes.indicesMapped[i] - if i < numIndicesMapped-1: - h += "," - h += " };\n" - else: - h += "#define Tensile_INDICES_MAPPED 0\n" - """ - - ############################################################################## - # Max Problem Sizes - ############################################################################## - if forBenchmark: - maximumD = problemSizes.maxD - maximumC = problemSizes.maxC - maximumA = problemSizes.maxA - maximumB = problemSizes.maxB - maximumW = problemSizes.maxD * 32; - - maxMT = getMaxSolutionSizes(solutions, solutionSummationSizes) - - maxMN = 1296 * maxMT[0] * maxMT[1] - maxMK = 36 * maxMT[0] * maxMT[2] - maxNK = 36 * maxMT[1] * maxMT[2] - - maximumA = max(maximumA, maxMK) - maximumB = max(maximumB, maxNK) - maximumC = max(maximumC, maxMN) - maximumD = max(maximumD, maxMN) - maximumW = max(maximumW, maxMN) - - h += "size_t maxSizeD = %u;\n" % (maximumD) - h += "size_t maxSizeC = %u;\n" % (maximumC) - h += "size_t maxSizeA = %u;\n" % (maximumA) - h += "size_t maxSizeB = %u;\n" % (maximumB) - h += "size_t maxSizeW = %u;\n" % (maximumW) - h += "\n" - else: - h += "size_t maxSizeD;\n" - h += "size_t maxSizeC;\n" - h += "size_t maxSizeA;\n" - h += "size_t maxSizeB;\n" - h += "size_t maxSizeW;\n" - h += "\n" - - ############################################################################## - # Current Problem Size - ############################################################################## - h += "/* current problem size */\n" - #h += "unsigned int fullSizes[maxNumIndices];\n" - #h += "unsigned int currentSizedIndexSizes[numIndicesSized];\n" - #h += "unsigned int currentSizedIndexIncrements[numIndicesSized];\n" - h += "\n" - - ############################################################################## - # Solutions - ############################################################################## - if forBenchmark: - # Solution Ptrs - h += "/* solutions */\n" - # Problem Type Indices - h += "const unsigned int maxNumSolutions = %u;\n" % len(solutions) - h += "float solutionPerf[numProblems][maxNumSolutions]; // milliseconds\n" - h += "\n" - - h += "static const SolutionInfo solutions[maxNumSolutions] = {\n" - for i in range(0, len(solutions)): - solution = solutions[i] - solutionName = solutionWriter.getSolutionName(solution) - h += " {(void*)%s, \"%s\", {%d, %d, %d, %d, %s} }" % \ - (solutionName, solutionName, - solution["AssertSummationElementMultiple"], - solution["AssertFree0ElementMultiple"], - solution["AssertFree1ElementMultiple"], - "false" - ) - if i < len(solutions)-1: - h += "," - h += "\n" - h += " };\n" - h += "\n" - - numSummations = len(solutionSummationSizes) - h += "const unsigned int numSummations = %d;\n" % (numSummations) - - h += "const unsigned int summations[numSummations] = {%d" % (solutionSummationSizes[0]) - for i in range(1, numSummations): - h += ", %d" % (solutionSummationSizes[i]) - h += "};\n" - - ############################################################################## - # Solution meta data - ############################################################################## - - transA = solutions[0]["ProblemType"]["TransposeA"] - transB = solutions[0]["ProblemType"]["TransposeB"] - h += "const unsigned int solutionMetaData[maxNumSolutions][10] = {\n" - for i in range(0, len(solutions)): - solution = solutions[i] - - wg = solution["WorkGroup"] - tt = solution["ThreadTile"] - mt0 = wg[0] * tt[0] - mt1 = wg[1] * tt[1] - gsu = solution["GlobalSplitU"] - lsu = wg[2] - - h += " {%d, %d, %d, %d, %d, %d, %d, %d, %d, %d}" % (mt0,mt1,tt[0],tt[1],wg[0],wg[1],transA,transB,gsu,lsu) - - if (i < len(solutions) - 1): - h += ",\n" - else: - h += "\n" - h += " };\n" - h += "\n" - - - - else: - # Function Names - functionNames = [] - for dataType in dataTypes: - for problemType in problemTypesForDataType[dataType]: - # example scheduleName is fiji, vega10, etc - for scheduleName in schedulesForProblemType[problemType]: - functionNames.append("tensile_%s" % (problemType)) - h += "const char *functionNames[numFunctions] = {\n" - for functionIdx in range(0, len(functionNames)): - functionName = functionNames[functionIdx] - h += " \"%s\"%s\n" % (functionName, \ - "," if functionIdx < len(functionNames)-1 else "" ) - h += " };\n" - - ############################################################################## - # Runtime Structures - ############################################################################## - h += "/* runtime structures */\n" - h += "TensileStatus status;\n" - if globalParameters["RuntimeLanguage"] == "OCL": - h += "cl_platform_id platform;\n" - h += "cl_device_id device;\n" - h += "cl_context context;\n" - h += "cl_command_queue stream;\n" - else: - h += "hipStream_t stream;\n" - #h += "int deviceIdx = %u;\n" \ - # % (globalParameters["Device"]) - h += "\n" - h += "void *deviceWS;\n" - h += "void *deviceD;\n" - h += "void *deviceC;\n" - h += "void *deviceA;\n" - h += "void *deviceB;\n" - - ############################################################################## - # Benchmarking and Validation Parameters - ############################################################################## - h += "\n/* benchmarking parameters */\n" - #h += "const bool measureKernelTime = %s;\n" \ - # % ("true" if globalParameters["KernelTime"] else "false") - #h += "const unsigned int numEnqueuesPerSync = %u;\n" \ - # % (globalParameters["EnqueuesPerSync"]) - #h += "const unsigned int numSyncsPerBenchmark = %u;\n" \ - # % (globalParameters["SyncsPerBenchmark"]) - #h += "unsigned int numElementsToValidate = %s;\n" \ - # % (str(globalParameters["NumElementsToValidate"]) \ - # if globalParameters["NumElementsToValidate"] >= 0 \ - # else "0xFFFFFFFF" ) - #h += "unsigned int validationMaxToPrint = %u;\n" \ - # % globalParameters["ValidationMaxToPrint"] - #h += "bool validationPrintValids = %s;\n" \ - # % ("true" if globalParameters["ValidationPrintValids"] else "false") - h += "size_t validationStride;\n" - if problemType["HighPrecisionAccumulate"]: - h += "static bool useHighPrecisionAccumulate = true;\n" - else: - h += "static bool useHighPrecisionAccumulate = false;\n" - #h += "unsigned int dataInitTypeC = %s;\n" % globalParameters["DataInitTypeC"] - #h += "unsigned int dataInitTypeAB = %s;\n" % globalParameters["DataInitTypeAB"] - h += "\n" - - ############################################################################## - # Generated Call to Reference - ############################################################################## - h += "/* generated call to reference */\n" - h += "template\n" - h += "TensileStatus generatedCallToReferenceCPU(\n" - h += " const unsigned int *sizes,\n" - h += " const unsigned int *minStrides,\n" - h += " DestDataType *referenceD,\n" - h += " DestDataType *referenceC,\n" - h += " DataType *initialA,\n" - h += " DataType *initialB,\n" - h += " const unsigned int lda,\n" - h += " const unsigned int ldb,\n" - h += " const unsigned int ldc,\n" - h += " const unsigned int ldd,\n" - h += " const unsigned int stride_a,\n" - h += " const unsigned int stride_b,\n" - h += " const unsigned int stride_c,\n" - h += " const unsigned int stride_d,\n" - h += " ComputeDataType alpha,\n" - h += " ComputeDataType beta,\n" - h += " bool useHighPrecisionAccumulate) {\n" - h += " return tensileReferenceCPU(\n" - h += " referenceD,\n" - h += " referenceC,\n" - h += " initialA,\n" - h += " initialB,\n" - h += " lda,\n" - h += " ldb,\n" - h += " ldc,\n" - h += " ldd,\n" - h += " stride_a,\n" - h += " stride_b,\n" - h += " stride_c,\n" - h += " stride_d,\n" - h += " alpha,\n" - h += " beta,\n" - h += " totalIndices[problemTypeIdx],\n" - h += " sizes,\n" - h += " minStrides,\n" - h += " numIndicesC[problemTypeIdx],\n" - h += " numIndicesA[problemTypeIdx],\n" - h += " numIndicesB[problemTypeIdx],\n" - h += " indexAssignmentsA[problemTypeIdx],\n" - h += " indexAssignmentsB[problemTypeIdx],\n" - h += " complexConjugateA[problemTypeIdx],\n" - h += " complexConjugateB[problemTypeIdx],\n" - h += " validationStride,\n" - h += " useHighPrecisionAccumulate);\n" - h += "};\n" - h += "\n" - - ############################################################################## - # Generated Call to Solution - ############################################################################## - if forBenchmark: - problemType = solutions[0]["ProblemType"] - h += "/* generated call to solution */\n" - h += "template\n" - h += "TensileStatus generatedCallToSolution(\n" - h += " const SolutionInfoType &solution,\n" - h += " SolutionLock *solutionLock,\n" - h += " const unsigned int *sizes,\n" - h += " const unsigned int *minStrides,\n" - h += " const unsigned int lda,\n" - h += " const unsigned int ldb,\n" - h += " const unsigned int ldc,\n" - h += " const unsigned int ldd,\n" - h += " const unsigned int stride_a,\n" - h += " const unsigned int stride_b,\n" - h += " const unsigned int stride_c,\n" - h += " const unsigned int stride_d,\n" - h += " ComputeDataType alpha,\n" - h += " ComputeDataType beta,\n" - h += " unsigned int numEvents = 0,\n" - if globalParameters["RuntimeLanguage"] == "OCL": - h += " cl_event *event_wait_list = NULL,\n" - h += " cl_event *outputEvent = NULL ) {\n" - else: - h += " hipEvent_t *startEvent = NULL,\n" - h += " hipEvent_t *stopEvent = NULL ) {\n" - - h += " // calculate parameters assuming packed data\n" - # strides - indexChars = globalParameters["IndexChars"] - firstStride = 1 - #assert(not problemType["UseInitialStridesCD"]) # not supported in old client - if problemType["UseInitialStridesAB"]: - firstStride = 0 - lastStrideD = problemType["NumIndicesC"] - lastStrideC = problemType["NumIndicesC"] - lastStrideA = len(problemType["IndexAssignmentsA"]) - lastStrideB = len(problemType["IndexAssignmentsB"]) - - # calculate strides - for i in range(0,lastStrideD): - h += " unsigned int strideD%u%s = 1" % (i, indexChars[i]) - for j in range(0, i): - h += " * (" - if j == 0: - h += "(ldd != std::numeric_limits::max()) ? ldd : " - h += "std::max(minStrides[%i], sizes[%i]))" % (j,j) - h += ";\n" - h += " if (stride_d != std::numeric_limits::max()) strideD%u%s = stride_d;\n" % (lastStrideD-1, indexChars[lastStrideD-1]) - for i in range(0,lastStrideC): - h += " unsigned int strideC%u%s = 1 " % (i, indexChars[i]) - for j in range(0, i): - h += " * (" - if j == 0: - h += "(ldc != std::numeric_limits::max()) ? ldc : " - h+= "std::max(minStrides[%i], sizes[%i]))" % (j,j) - h += ";\n" - h += " if (stride_c != std::numeric_limits::max()) strideC%u%s = stride_c;\n" % (lastStrideC-1, indexChars[lastStrideC-1]) - - constStride = None - for i in range(0,lastStrideA): - idx = problemType["IndexAssignmentsA"][i] - constStride = checkConstStride(problemType["SetConstStrideA"], idx) - if constStride != None: - h += " unsigned int strideA%u%s = %d; //SetConstStrideA\n" % (i, - indexChars[problemType["IndexAssignmentsA"][i]], - constStride) - else: - h += " unsigned int strideA%u%s = 1" % (i, \ - indexChars[problemType["IndexAssignmentsA"][i]]) - for j in range(0, i): - h += " * (" - if j == 0: - h += "(lda != std::numeric_limits::max()) ? lda : " - h += "std::max(minStrides[%i], sizes[%i]))" % \ - (problemType["IndexAssignmentsA"][j], - problemType["IndexAssignmentsA"][j]) - h += ";\n" - if constStride == None: - h += " if (stride_a != std::numeric_limits::max()) strideA%u%s = stride_a;\n" % (lastStrideA-1, indexChars[problemType["IndexAssignmentsA"][lastStrideA-1]]) - - for i in range(0,lastStrideB): - idx = problemType["IndexAssignmentsB"][i] - constStride = checkConstStride(problemType["SetConstStrideB"], idx) - if constStride != None: - h += " unsigned int strideB%u%s = %d; //SetConstStrideB\n" % (i, - indexChars[problemType["IndexAssignmentsB"][i]], - constStride) - else: - h += " unsigned int strideB%u%s = 1" % (i, \ - indexChars[problemType["IndexAssignmentsB"][i]]) - for j in range(0, i): - h += " * (" - if j == 0: - h += "(ldb != std::numeric_limits::max()) ? ldb : " - h+= "std::max(minStrides[%i], sizes[%i]))" % \ - (problemType["IndexAssignmentsB"][j], - problemType["IndexAssignmentsB"][j]) - h += ";\n" - h += " if (stride_b != std::numeric_limits::max()) strideB%u%s = stride_b;\n" % (lastStrideB-1, indexChars[problemType["IndexAssignmentsB"][lastStrideB-1]]) - - for i in range(0, problemType["TotalIndices"]): - h += " unsigned int size%s = sizes[%u];\n" % (indexChars[i], i) - h += "\n" - - - # function call - h += " // Check assertions,\n" - assert(not problemType["UseInitialStridesCD"]) # not supported in old client - firstStride = 0 if problemType["UseInitialStridesAB"] else 1 - lastStrideD = problemType["NumIndicesC"] - lastStrideC = problemType["NumIndicesC"] - lastStrideA = len(problemType["IndexAssignmentsA"]) - lastStrideB = len(problemType["IndexAssignmentsB"]) - numSizes = problemType["TotalIndices"] - h += " typedef ProblemDims<%u,%u,%u,%u,%u,%u> ProblemDims_%s;\n" \ - % (firstStride, lastStrideD, lastStrideC, lastStrideA, lastStrideB, numSizes, problemType) - # TODO - this should be initialized somewhere once? - h += " static const ProblemType problemType( " - h += listToInitializer(problemType["IndicesFree"]) + ", " - h += listToInitializer(problemType["IndicesSummation"]) + ", " - h += listToInitializer(problemType["IndicesBatch"]) + ', ' - h += listToInitializer(problemType["IndexAssignmentsA"]) + ', ' - h += listToInitializer(problemType["IndexAssignmentsB"]) - h += ");\n" - # create problem size - TODO could move this up to the caller - h += " ProblemDims_%s pdims(" % problemType - indexChars = globalParameters["IndexChars"] - for i in range(firstStride,lastStrideD): - if i != firstStride: h += ", " - h += "strideD%u%s" % (i, indexChars[i]) - for i in range(firstStride,lastStrideC): - h += ", strideC%u%s" % (i, indexChars[i]) - for i in range(firstStride,lastStrideA): - h += ", strideA%u%s" % (i, \ - indexChars[problemType["IndexAssignmentsA"][i]]) - for i in range(firstStride,lastStrideB): - h += ", strideB%u%s" % (i, \ - indexChars[problemType["IndexAssignmentsB"][i]]) - for i in range(0, problemType["TotalIndices"]): - h += ", size%s" % indexChars[i] - h += ");\n" - h += " if (!ProblemProperties(pdims,&problemType).validForSolution(solution._assertionRequirements))\n" - h += " return tensileStatusAssertFailure; // problem dims did not meet requirements for solution\n" - h += "\n" - - h += " // call solution function\n" - h += " TensileSolutionPointer_%s f = reinterpret_cast (solution._functionPtr);\n" \ - % (problemType, problemType) - if globalParameters["RuntimeLanguage"] == "OCL": - h += " return f(solutionLock, static_cast(deviceD), static_cast(deviceC), static_cast(deviceA), static_cast(deviceB),\n" - else: - typeName = dataTypes[0].toCpp() - destTypeName = destDataTypes[dataType].toCpp() - computeTypeName = computeDataTypes[dataType].toCpp() - h += " return f(solutionLock,\n" - h += " static_cast<%s *>(deviceD),\n" % destTypeName - h += " static_cast<%s *>(deviceC),\n" % destTypeName - h += " static_cast<%s *>(deviceA),\n" % typeName - h += " static_cast<%s *>(deviceB),\n" % typeName - h += " alpha,\n" - if problemType["UseBeta"]: - h += " beta,\n" - for i in range(firstStride,lastStrideD): - h += " strideD%u%s,\n" % (i, indexChars[i]) - for i in range(firstStride,lastStrideC): - h += " strideC%u%s,\n" % (i, indexChars[i]) - for i in range(firstStride,lastStrideA): - h += " strideA%u%s,\n" % (i, \ - indexChars[problemType["IndexAssignmentsA"][i]]) - for i in range(firstStride,lastStrideB): - h += " strideB%u%s,\n" % (i, \ - indexChars[problemType["IndexAssignmentsB"][i]]) - for i in range(0, problemType["TotalIndices"]): - h += " size%s,\n" % indexChars[i] - h += " stream,\n" - if globalParameters["RuntimeLanguage"] == "OCL": - h += " numEvents, event_wait_list, outputEvent ); // events\n" - else: - h += " numEvents,\n" - h += " startEvent,\n" - h += " stopEvent,\n" - h += " static_cast(deviceWS)); // events\n" - - h += "};\n" - h += "\n" - else: - ############################################################################ - # Generated Call to Function - ############################################################################ - for enqueue in [True, False]: - functionName = "tensile" if enqueue else "tensileGetSolutionName" - returnName = "TensileStatus" if enqueue else "const char *" - h += "/* generated call to function */\n" - h += "template\n" - h += "%s generatedCallTo_%s(\n" % (returnName, functionName) - h += " unsigned int *sizes,\n" - h += " unsigned int *minStrides,\n" - h += " ComputeDataType alpha,\n" - h += " ComputeDataType beta,\n" - h += " unsigned int lda,\n" - h += " unsigned int ldb,\n" - h += " unsigned int ldc,\n" - h += " unsigned int ldd,\n" - h += " unsigned int strideA,\n" - h += " unsigned int strideB,\n" - h += " unsigned int strideC,\n" - h += " unsigned int strideD,\n" - h += " unsigned int numEvents = 0,\n" - - if globalParameters["RuntimeLanguage"] == "OCL": - h += " cl_event *event_wait_list = NULL,\n" - h += " cl_event *outputEvent = NULL );\n\n" - else: - h += " hipEvent_t *startEvent = NULL,\n" - h += " hipEvent_t *stopEvent = NULL );\n\n" - - -#need to get DestDataType in here - for dataType in dataTypes: - typeName = dataType.toCpp() - destDataType = destDataTypes[dataType] - destTypeName = destDataType.toCpp() - computeDataType = computeDataTypes[dataType] - computeTypeName = computeDataType.toCpp() - functionsForDataType = [] - for problemType in problemTypesForDataType[dataType]: - for scheduleName in schedulesForProblemType[problemType]: - functionsForDataType.append([scheduleName, problemType]) - h += "template<>\n" - h += "inline %s generatedCallTo_%s<%s, %s, %s>(\n" \ - % (returnName, functionName, typeName, destTypeName, computeTypeName) - h += " unsigned int *sizes,\n" - h += " unsigned int *minStrides,\n" - h += " %s alpha,\n" % computeTypeName - h += " %s beta,\n" % computeTypeName - h += " unsigned int lda,\n" - h += " unsigned int ldb,\n" - h += " unsigned int ldc,\n" - h += " unsigned int ldd,\n" - h += " unsigned int strideA,\n" - h += " unsigned int strideB,\n" - h += " unsigned int strideC,\n" - h += " unsigned int strideD,\n" - h += " unsigned int numEvents, \n" - - if globalParameters["RuntimeLanguage"] == "OCL": - h += " cl_event *event_wait_list,\n" - h += " cl_event *outputEvent ) {\n\n" - else: - h += " hipEvent_t *startEvent,\n" - h += " hipEvent_t *stopEvent ) {\n\n" - - h += " unsigned int functionIdxForDataType = functionInfo[functionIdx][4];\n" - - for functionIdx in range(0, len(list(functionsForDataType))): - function = functionsForDataType[functionIdx] - scheduleName = function[0] - problemType = function[1] - if len(list(functionsForDataType))> 1: - if functionIdx == 0: - h += " if (functionIdxForDataType == %u) {\n" % functionIdx - elif functionIdx == len(list(functionsForDataType))-1: - h += " } else {\n" - else: - h += " } else if (functionIdxForDataType == %u) {\n" \ - % functionIdx - - # strides - indexChars = globalParameters["IndexChars"] - firstStride = 1 - assert(not problemType["UseInitialStridesCD"]) # not supported in old client - if problemType["UseInitialStridesAB"]: - firstStride = 0 - lastStrideD = problemType["NumIndicesC"] - lastStrideC = problemType["NumIndicesC"] - lastStrideA = len(problemType["IndexAssignmentsA"]) - lastStrideB = len(problemType["IndexAssignmentsB"]) - - # calculate strides - for i in range(0,lastStrideD): - h += " unsigned int strideD%u%s = 1" % (i, indexChars[i]) - for j in range(0, i): - h += "*sizes[%i]" % j - h += ";\n" - h += " if (strideD != std::numeric_limits::max()) strideD%u%s = strideD;\n" % (lastStrideD-1, indexChars[lastStrideD-1]) - for i in range(0,lastStrideC): - h += " unsigned int strideC%u%s = 1" % (i, indexChars[i]) - for j in range(0, i): - h += "*sizes[%i]" % j - h += ";\n" - h += " if (strideC != std::numeric_limits::max()) strideC%u%s = strideC;\n" % (lastStrideC-1, indexChars[lastStrideC-1]) - - for i in range(0,lastStrideA): - h += " unsigned int strideA%u%s = 1" % (i, \ - indexChars[problemType["IndexAssignmentsA"][i]]) - for j in range(0, i): - h += "*sizes[%i]" % \ - problemType["IndexAssignmentsA"][j] - h += ";\n" - h += " if (strideA != std::numeric_limits::max()) strideA%u%s = strideA;\n" % (lastStrideA-1, indexChars[problemType["IndexAssignmentsA"][lastStrideA-1]]) - for i in range(0,lastStrideB): - h += " unsigned int strideB%u%s = 1" % (i, \ - indexChars[problemType["IndexAssignmentsB"][i]]) - for j in range(0, i): - h += "*sizes[%i]" % \ - problemType["IndexAssignmentsB"][j] - h += ";\n" - h += " if (strideB != std::numeric_limits::max()) strideB%u%s = strideB;\n" % (lastStrideB-1, indexChars[problemType["IndexAssignmentsB"][lastStrideB-1]]) - for i in range(0, problemType["TotalIndices"]): - h += " unsigned int size%s = sizes[%u];\n" % (indexChars[i], i) - - # function call - h += " // call solution function\n" - h += " return %s_%s(\n" % (functionName, problemType) - if enqueue: - if globalParameters["RuntimeLanguage"] == "OCL": - h += " static_cast(deviceD),\n" - h += " static_cast(deviceC),\n" - h += " static_cast(deviceA),\n" - h += " static_cast(deviceB),\n" - else: - h += " static_cast<%s *>(deviceD),\n" % destTypeName - h += " static_cast<%s *>(deviceC),\n" % destTypeName - h += " static_cast<%s *>(deviceA),\n" % typeName - h += " static_cast<%s *>(deviceB),\n" % typeName - h += " alpha,\n" - if problemType["UseBeta"]: - h += " beta,\n" - for i in range(firstStride,lastStrideD): - h += " strideD%u%s,\n" % (i, indexChars[i]) - for i in range(firstStride,lastStrideC): - h += " strideC%u%s,\n" % (i, indexChars[i]) - for i in range(firstStride,lastStrideA): - h += " strideA%u%s,\n" % (i, \ - indexChars[problemType["IndexAssignmentsA"][i]]) - for i in range(firstStride,lastStrideB): - h += " strideB%u%s,\n" % (i, \ - indexChars[problemType["IndexAssignmentsB"][i]]) - for i in range(0, problemType["TotalIndices"]): - h += " size%s%s\n" % (indexChars[i], "," if i != problemType["TotalIndices"]-1 else "") - if enqueue: - if globalParameters["RuntimeLanguage"] == "OCL": - h += ", stream, numEvents, event_wait_list, outputEvent" - else: - h += ", stream, numEvents, startEvent, stopEvent, static_cast(deviceWS)" - h += ");\n" - - if len(functionsForDataType) > 1: - h += " }\n" # close last if - h += "};\n" # close callToFunction - - ############################################################################## - # Results File Name - ############################################################################## - if forBenchmark: - h += "/* results file name */\n" - resultsFileName = os.path.join(stepBaseDir, \ - "../Data","%s.csv" % stepName) - resultsFileName = resultsFileName.replace("\\", "\\\\") - h += "const char *resultsFileName = \"%s\";\n" % resultsFileName - - granularityFileName = os.path.join(stepBaseDir, \ - "../Data","%s_Granularity.csv" % stepName) - - granularityFileName = granularityFileName.replace("\\", "\\\\") - h += "const char *granularityFileName = \"%s\";\n" % granularityFileName - - ############################################################################## - # Write File - ############################################################################## - clientParametersFile = open(os.path.join(globalParameters["WorkingPath"], \ - "ClientParameters.h"), "w") - clientParametersFile.write(CHeader) - clientParametersFile.write(h) - clientParametersFile.close() diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 115eeffd8f..5b7e1c128c 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -35,7 +35,7 @@ import sys import time import re - + IsaVersion = Tuple[int, int, int] @@ -241,7 +241,9 @@ globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi +# deleteme globalParameters["WorkingPath"] = os.getcwd() # path where tensile called from +# deleteme globalParameters["IndexChars"] = "IJKLMNOPQRSTUVWXYZ" # which characters to use for C[ij]=Sum[k] A[ik]*B[jk] globalParameters["ScriptPath"] = os.path.dirname(os.path.realpath(__file__)) # path to Tensile/Tensile.py globalParameters["SourcePath"] = os.path.join(globalParameters["ScriptPath"], "Source") # path to Tensile/Source/ @@ -1683,7 +1685,7 @@ def assignGlobalParameters(config, cxxCompiler=None): # ROCm Agent Enumerator Path if os.name == "nt": globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") - globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") + globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") else: globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch") globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator") @@ -1788,6 +1790,7 @@ def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ else: destinationDictionary[key] = deepcopy(defaultDictionary[key]) +# deleteme ################################################################################ # Push / Pop Working Path # store a WorkingPath where to write files (like benchmark files) @@ -1804,6 +1807,13 @@ def popWorkingPath(): os.path.split(globalParameters["WorkingPath"])[0] else: globalParameters["WorkingPath"] = workingDirectoryStack.pop() +def setWorkingPath( fullPathName ): + # Warning: this is not thread-safe, modifies the global WorkingPath! + workingDirectoryStack.append(globalParameters["WorkingPath"]) + globalParameters["WorkingPath"] = ensurePath(fullPathName) +# deleteme + + def ensurePath(path): try: os.makedirs(path) @@ -1812,10 +1822,6 @@ def ensurePath(path): except OSError: printExit("Failed to create directory \"%s\" " % (path) ) return path -def setWorkingPath( fullPathName ): - # Warning: this is not thread-safe, modifies the global WorkingPath! - workingDirectoryStack.append(globalParameters["WorkingPath"]) - globalParameters["WorkingPath"] = ensurePath(fullPathName) def roundUp(f): diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 4f71179d51..5f2cb8655e 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ # ################################################################################ +from pathlib import Path from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ setWorkingPath, popWorkingPath, assignParameterWithDefault, startTime, ProgressBar, printWarning from .SolutionStructs import Solution @@ -40,7 +41,7 @@ ################################################################################ # Analyze Problem Type ################################################################################ -def analyzeProblemType( problemType, problemSizeGroups, inputParameters ): +def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryLogicPath): print2(HR) print1("# Analyzing: %s" % problemType) @@ -190,7 +191,7 @@ def analyzeProblemType( problemType, problemSizeGroups, inputParameters ): permutations.append(permutation) #print permutations for permutation in permutations: - logicAnalyzer.print2D(permutation) + logicAnalyzer.print2D(permutation, libraryLogicPath) ###################################### # Range Logic @@ -463,7 +464,7 @@ def addFromCSV(self, dataFileName, numSolutions, solutionMap): else: printWarning("Performance unit %s in %s is unrecognized: assuming GFlops (device efficiency)" % (perfUnit, dataFileName)) self.perfMetric = "DeviceEfficiency" - + if csvHasWinner: try: # These two columns only appear when using WinnerCSV @@ -472,14 +473,14 @@ def addFromCSV(self, dataFileName, numSolutions, solutionMap): except ValueError as e: csvHasWinnerColumn = False print1(f"Error: Could not find WinnerGFlops or WinnerIdx column in CSV file: {e}") - + # get the column index of Frequency(MHz) try: columnOfFreqIdx = row.index(" DeviceMaxFreq") except ValueError as e: columnOfFreqIdx = None print1(f"Error: Could not find DeviceMaxFreq column in the CSV file: {e}") - + # get the length of each row, and derive the first column of the solution instead of using wrong "solutionStartIdx = totalSizeIdx + 1" rowLength = len(row) solutionStartIdx = rowLength - numSolutions @@ -514,7 +515,7 @@ def addFromCSV(self, dataFileName, numSolutions, solutionMap): winnerIdx = solutionIdx winnerGFlops = gflops solutionIdx += 1 - + if globalParameters["UseEffLike"]: try: frequency = float(row[columnOfFreqIdx]) @@ -530,7 +531,7 @@ def addFromCSV(self, dataFileName, numSolutions, solutionMap): performance_metric = float(winnerGFlops) else: performance_metric = float(winnerGFlops) - + if winnerIdx != -1: if problemSize in self.exactWinners: if winnerGFlops > self.exactWinners[problemSize][1]: @@ -539,7 +540,7 @@ def addFromCSV(self, dataFileName, numSolutions, solutionMap): else: self.exactWinners[problemSize] = [solutionMap[winnerIdx], performance_metric] #print "new exact", problemSize, "CSV index=", winnerIdx, self.exactWinners[problemSize] - + # Range Problem Size elif problemSize in self.rangeProblemSizes: problemIndices = [] @@ -938,7 +939,7 @@ def prepareLogic(self, logic): ############################################################################## # Print2D ############################################################################## - def print2D(self, indices ): + def print2D(self, indices, libraryLogicPath): indicesIdx = 0 problemIndices = [] for i in range(0, self.numIndices): @@ -1027,7 +1028,7 @@ def print2D(self, indices ): for idx in indices: printFileName += "_%u" % idx printFileName += ".csv" - printFile = open(os.path.join(globalParameters["WorkingPath"], printFileName), "w") + printFile = open(os.path.join(libraryLogicPath, printFileName), "w") printFile.write( w ) printFile.write( g ) printFile.write( f ) @@ -1454,7 +1455,10 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) print2("# LibraryLogic config: %s" % config) print2("# DefaultAnalysisParameters: " % defaultAnalysisParameters) + # deleteme setWorkingPath(libraryLogicPath) + assert libraryLogicPath == globalParameters["WorkingPath"], f"Lib logic path: {libraryLogicPath} benchmark data path: {benchmarkDataPath}" + # deleteme # Assign Defaults analysisParameters = {} @@ -1502,9 +1506,9 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) dataFileName, solutionsFileName, selectionFileName, solutions) ) for problemType in problemTypes: - logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters) + logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters, libraryLogicPath) - filename = os.path.join(globalParameters["WorkingPath"], \ + filename = os.path.join(libraryLogicPath, \ "{}_{}".format(analysisParameters["ScheduleName"], str(problemType))) print2("# writing library logic YAML {}".format(filename)) @@ -1521,10 +1525,13 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) currentTime = time.time() elapsedTime = currentTime - startTime print1("%s\n# Finish Analysing data to %s in %.3fs\n%s" % (HR, os.path.split(libraryLogicPath)[0], elapsedTime, HR) ) + + # deleteme popWorkingPath() + # deleteme ############################################################################## -# Error handling for frequency issues +# Error handling for frequency issues ############################################################################## def handle_frequency_issue(message): print1(message) @@ -1542,7 +1549,7 @@ def handle_frequency_issue(message): raise Exception("User chose to abort due to frequency issue.") else: globalParameters["UseEffLike"] = False - print1("Proceeding with GFlops as the efficiency metric.") + print1("Proceeding with GFlops as the efficiency metric.") ################################################################################ ################################################################################ ### @@ -1550,12 +1557,26 @@ def handle_frequency_issue(message): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str): +def main(config, cxxCompiler: str, outputPath: Path): + + benchmarkDataPath = outputPath / globalParameters["BenchmarkDataPath"] - benchmarkDataPath = os.path.join(globalParameters["WorkingPath"], \ + # deleteme + benchmarkDataPathOld = os.path.join(globalParameters["WorkingPath"], \ globalParameters["BenchmarkDataPath"]) + assert benchmarkDataPathOld == benchmarkDataPath, f"benchmarkDataPathOld: {benchmarkDataPathOld}, benchmarkDataPath: {benchmarkDataPath}" + # deleteme - libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ + libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] + + # deleteme + libraryLogicPathOld = os.path.join(globalParameters["WorkingPath"], \ globalParameters["LibraryLogicPath"]) + assert libraryLogicPathOld == libraryLogicPath, f"libraryLogicPathOld: {libraryLogicPathOld}, libraryLogicPath: {libraryLogicPath}" + # deleteme generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) + + # deleteme + assert globalParameters["WorkingPath"] == outputPath, f"WorkingPath: {globalParameters['WorkingPath']}, outputPath: {outputPath}" + # deleteme diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 2d8bef27f4..a335b9fb61 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -51,9 +51,40 @@ # LibraryLogic.main() to analyse final benchmark data and produce logic/yaml # ClientWriter.main() to create client which calls library based on above yaml ################################################################################ -def executeStepsInConfig(config, outputPath: Path, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str): +def executeStepsInConfig( + config: dict, + outputPath: Path, + asmToolchain: AssemblyToolchain, + srcToolchain: SourceToolchain, + cCompiler: str + ): + """Conducts the steps in the provided ``config`` according to the Tensile workflow. + + The top-level steps are: + 1. BenchmarkProblems: Runs the benchmarking steps and generates the directories + build_tmp, 1_BenchmarkProblems, 2_BenchmarkData + 2. LibraryLogic: Analyzes the benchmark data, makes logic files, and generates + the directory 3_LibraryLogic + 3. LibraryClient: Makes the client callable libraries and generates the + directory 4_LibraryClient + + Args: + config (dict): The configuration dictionary. + outputPath (Path): The path to the top-level build directory. + asmToolchain (AssemblyToolchain): The toolchain for making assembly kernels. + srcToolchain (SourceToolchain): The toolchain for making source kernels. + cCompiler (str): The C compiler to use. + """ + # Goal, define these here and pass them in to the relevant functions to it's clear + # what is being built and where buildTmpPath = outputPath / "build_tmp" + clientBuildPath = outputPath / "0_Build" + benchamrkProblemsPath = outputPath / "1_BenchmarkProblems" + benchmarkDataPath = outputPath / "2_BenchmarkData" + libraryLogicPath = outputPath / "3_LibraryLogic" + clientLibraryPath = outputPath / "4_LibraryClient" + ############################################################################## # Benchmark Problems ############################################################################## @@ -75,7 +106,7 @@ def executeStepsInConfig(config, outputPath: Path, asmToolchain: AssemblyToolcha libraryLogicConfig = config["LibraryLogic"] else: libraryLogicConfig = {} - LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler) + LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath) print1("") else: print1("# LibraryLogic already done.") @@ -281,8 +312,11 @@ def Tensile(userArgs): srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) globalParameters["OutputPath"] = outputPath + + # deleteme globalParameters["WorkingPath"] = outputPath print("WorkingPath: %s" % globalParameters["WorkingPath"]) + # deleteme overrideParameters = argUpdatedGlobalParameters(args) diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py index b4464e99ed..4d9a51d8e9 100644 --- a/tensilelite/Tensile/TensileRetuneLibrary.py +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -1,6 +1,6 @@ ############################################################################### # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -75,6 +75,8 @@ def runBenchmarking(solutions, problemSizes, outPath, update, cxxCompiler: str, # could use a refactor to elimate duplicated code ClientExecutable.getClientExecutable(cxxCompiler, cCompiler) + + shortName = "benchmark" benchmarkDir = os.path.join(outPath, shortName) sourceDir = os.path.join(benchmarkDir, "source") @@ -161,9 +163,12 @@ def TensileRetuneLibrary(userArgs): ############################################## outPath = ensurePath(os.path.abspath(args.OutputPath)) restoreDefaultGlobalParameters() + + # deleteme -- "WorkingPath" assignGlobalParameters({"LibraryFormat": "msgpack", "OutputPath": outPath, "WorkingPath": outPath}) + # deleteme overrideParameters = argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): From 17aa51a6ce44b320e50673013014fd95bb94ea97 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 17 Jan 2025 19:35:57 +0000 Subject: [PATCH 005/105] fix: string wrap paths for assert checks --- tensilelite/Tensile/LibraryLogic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 5f2cb8655e..78ec4e97ff 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -1564,7 +1564,7 @@ def main(config, cxxCompiler: str, outputPath: Path): # deleteme benchmarkDataPathOld = os.path.join(globalParameters["WorkingPath"], \ globalParameters["BenchmarkDataPath"]) - assert benchmarkDataPathOld == benchmarkDataPath, f"benchmarkDataPathOld: {benchmarkDataPathOld}, benchmarkDataPath: {benchmarkDataPath}" + assert benchmarkDataPathOld == str(benchmarkDataPath), f"benchmarkDataPathOld: {benchmarkDataPathOld}, benchmarkDataPath: {benchmarkDataPath}" # deleteme libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] @@ -1572,11 +1572,11 @@ def main(config, cxxCompiler: str, outputPath: Path): # deleteme libraryLogicPathOld = os.path.join(globalParameters["WorkingPath"], \ globalParameters["LibraryLogicPath"]) - assert libraryLogicPathOld == libraryLogicPath, f"libraryLogicPathOld: {libraryLogicPathOld}, libraryLogicPath: {libraryLogicPath}" + assert libraryLogicPathOld == str(libraryLogicPath), f"libraryLogicPathOld: {libraryLogicPathOld}, libraryLogicPath: {libraryLogicPath}" # deleteme generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) # deleteme - assert globalParameters["WorkingPath"] == outputPath, f"WorkingPath: {globalParameters['WorkingPath']}, outputPath: {outputPath}" + assert globalParameters["WorkingPath"] == str(outputPath), f"WorkingPath: {globalParameters['WorkingPath']}, outputPath: {outputPath}" # deleteme From 66c157d2b7c275270969270c88a635d83e84a7a8 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Sat, 18 Jan 2025 01:28:43 +0000 Subject: [PATCH 006/105] fix: ensure all paths are built --- tensilelite/Tensile/BenchmarkProblems.py | 67 +------------------ tensilelite/Tensile/ClientWriter.py | 65 ++---------------- tensilelite/Tensile/Common.py | 27 -------- tensilelite/Tensile/LibraryLogic.py | 32 +-------- tensilelite/Tensile/Tensile.py | 8 +-- .../Tensile/TensileCreateLibrary/Run.py | 3 +- tensilelite/Tensile/TensileRetuneLibrary.py | 56 ++++++++++++---- 7 files changed, 53 insertions(+), 205 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 808288ec19..e3f0fd54d3 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -38,7 +38,7 @@ from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni -from .Common import globalParameters, HR, pushWorkingPath, popWorkingPath, print1, print2, \ +from .Common import globalParameters, HR, print1, print2, \ printExit, printWarning, ensurePath, startTime, validParameters from .KernelWriterAssembly import KernelWriterAssembly from .SolutionStructs import Solution, ProblemType, ProblemSizes @@ -120,10 +120,6 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ ensurePath(sourcePath / "Solutions") ensurePath(sourcePath / "Kernels") - # deleteme - assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"outputPath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - # deleteme - copyStaticFiles(sourcePath) kernels = [] @@ -161,12 +157,6 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') - - # deleteme - newLibraryDirOld = ensurePath(os.path.join(globalParameters["WorkingPath"], 'library')) - assert newLibraryDirOld == str(newLibraryDir), f"newLibraryDirOld={newLibraryDirOld} newLibraryDir={newLibraryDir}" - # deleteme - newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, srcToolchain.compiler) newLibrary.applyNaming(kernelMinNaming) @@ -227,16 +217,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG enableTileSelection = benchmarkProcess.problemType["TileAwareSelection"] groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) - groupNamePath = benchmarkProblemsPath / groupName - - # deleteme - pushWorkingPath(groupName) - assert str(groupNamePath) == globalParameters["WorkingPath"], f"Group name working path: {globalParameters['WorkingPath']} and the group name path: {groupNamePath}" - - ensurePath(os.path.join(globalParameters["WorkingPath"], "Data")) - # deleteme - ensurePath(groupNamePath / "Data") totalBenchmarkSteps = len(benchmarkProcess) @@ -269,23 +250,10 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG if benchmarkStep.internalSupportParams: print("# InternalSupportParams: {}".format(benchmarkStep.internalSupportParams)) - shortNamePath = groupNamePath / shortName - # deleteme - pushWorkingPath(shortName) - assert str(shortNamePath) == globalParameters["WorkingPath"], f"shortNamePath={shortNamePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - # deleteme - + shortNamePath = ensurePath(groupNamePath / shortName) stepBaseDir = shortNamePath - - # file paths resultsFileBase = os.path.normpath(shortNamePath / ".." / "Data" / shortName) - # deleteme - resultsFileBaseOld = os.path.normpath(os.path.join( \ - globalParameters["WorkingPath"], "../Data", shortName)) - assert resultsFileBaseOld == resultsFileBase, f"resultsFileBaseOld={resultsFileBaseOld} resultsFileBase={resultsFileBase}" - # deleteme - if benchmarkStep.isFinal(): resultsFileBaseFinal = resultsFileBase resultsFileName = resultsFileBase + ".csv" @@ -295,11 +263,6 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG cachePath = os.path.join(stepBaseDir, "cache.yaml") sourcePath = ensurePath(shortNamePath / "source") - # deleteme - pushWorkingPath("source") - assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - # deleteme - cacheValid = False if useCache and os.path.isfile(cachePath): c = LibraryIO.read(cachePath) @@ -386,11 +349,6 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) outFile = os.path.join(sourcePath, "ClientParameters.ini") - # deleteme - assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"stepBaseDir={stepBaseDir / 'source'} globalParameters[WorkingPath]={globalParameters['WorkingPath']}" - # deleteme - writeClientConfigIni(True, benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, benchmarkStep.icacheFlushArgs, conProblemType, @@ -402,10 +360,6 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG LibraryIO.writeSolutions(solutionsFileName, benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, benchmarkStep.activationArgs, solutions, cacheValid) - # deleteme - popWorkingPath() # source - # deleteme - # run benchmarking client if not os.path.exists(resultsFileName) or globalParameters["ForceRedoBenchmarkProblems"]: libraryLogicPath = None @@ -420,16 +374,10 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG print1("# Already benchmarked; skipping.") # End Iteration - # deleteme - popWorkingPath() # stepName - # deleteme currentTime = time.time() elapsedTime = currentTime - startTime print1("{}\n# {}\n# {}: End - {:.3f}s\n{}\n" \ .format(HR, groupName, shortName, elapsedTime, HR)) - # deleteme - popWorkingPath() # ProblemType - # deleteme return (resultsFileBaseFinal, benchmarkTestFails) @@ -444,13 +392,6 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source benchmarkDataPath = ensurePath(outputPath / globalParameters["BenchmarkDataPath"]) - # deleteme - dataPathOld = os.path.join(globalParameters["WorkingPath"], globalParameters["BenchmarkDataPath"]) - pushWorkingPath(globalParameters["BenchmarkProblemsPath"]) - assert dataPathOld == str(benchmarkDataPath), f"dataPathOld={dataPathOld} dataPath={benchmarkDataPath}" - # deleteme - - totalTestFails = 0 for benchmarkProblemTypeConfig in config: problemTypeConfig = benchmarkProblemTypeConfig[0] @@ -501,9 +442,5 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source print1("# {}_{:02d} already benchmarked; skipping." \ .format(str(problemTypeObj), idx) ) - # deleteme - popWorkingPath() # BenchmarkProblemsPath - # deleteme - if globalParameters["ExitOnFails"] and totalTestFails: sys.exit(1) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index e4f47190aa..22b3683e7d 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -26,7 +26,7 @@ from . import ClientExecutable from . import LibraryIO from .TensileInstructions import getGfxName, DataType -from .Common import globalParameters, pushWorkingPath, popWorkingPath, print1, printExit, CHeader, printWarning, listToInitializer, ClientExecutionLock +from .Common import globalParameters, ensurePath, print1, printExit, CHeader, printWarning, listToInitializer, ClientExecutionLock from .SolutionStructs import Problem, ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles @@ -80,25 +80,9 @@ class ClientLogLevel(Enum): ################################################################################ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): - libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] - # deleteme - libraryLogicPath = os.path.join(globalParameters["WorkingPath"], \ - globalParameters["LibraryLogicPath"]) - assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" - # deleteme - - clientLibraryPath = outputPath / globalParameters["LibraryClientPath"] - # deleteme - stepBaseDirOld = pushWorkingPath(globalParameters["LibraryClientPath"]) - assert str(clientLibraryPath) == stepBaseDirOld, f"stepBaseDir={clientLibraryPath} != SBD={stepBaseDirOld}" - # deleteme - - sourcePath = clientLibraryPath / "source" - # deleteme - pushWorkingPath("source") - assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" - # deleteme - + libraryLogicPath = ensurePath(outputPath / globalParameters["LibraryLogicPath"]) + clientLibraryPath = ensurePath(outputPath / globalParameters["LibraryClientPath"]) + sourcePath = ensurePath(clientLibraryPath / "source") copyStaticFiles(sourcePath) ############################################################################## @@ -155,11 +139,6 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): activationArgs = ActivationArgs(problemType, activationEnums) if isForAll else "" factorDimArgs = FactorDimArgs(problemType, factorDimEnums) - # deleteme - assert str(sourcePath) == globalParameters["WorkingPath"], f"sourcePath={sourcePath} != WP={globalParameters['WorkingPath']}" - print1(f"sourcePath={sourcePath} == WP={globalParameters['WorkingPath']}") - # deleteme - clientParametersPaths.append(writeClientConfig( forBenchmark=False, solutions=None, @@ -177,25 +156,14 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): libraryFile=yamlList[0])) globalParameters["EnableHalf"] = enableHalf - ############################################################################## - # Write Generated Header - ############################################################################## forBenchmark = False problemSizes = None - # deleteme - popWorkingPath() # source - # deleteme - ############################################################################## # Run Build Script ############################################################################## # if redo=true, clobber the build directory - # deleteme - assert str(clientLibraryPath) == globalParameters["WorkingPath"], f"clientLibraryPath={clientLibraryPath} != WP={globalParameters['WorkingPath']}" - # deleteme - if globalParameters["ForceRedoLibraryClient"]: shutil.rmtree(os.path.join(clientLibraryPath, "build"), \ ignore_errors=True) @@ -204,11 +172,6 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): enableTileSelection = False returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, outputPath, clientParametersPaths) - # deleteme - popWorkingPath() # LibraryClient - assert str(outputPath) == globalParameters["WorkingPath"], f"outputPath={outputPath} != WP={globalParameters['WorkingPath']}" - # deleteme - return returncode ################################################################################ @@ -227,14 +190,8 @@ def runNewClient(scriptPath, clientParametersPath, cxxCompiler: str, cCompiler: def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, outputPath, configPaths=None): - # write runScript - buildPath = outputPath / "build" - # deleteme - pushWorkingPath("build") - wkpath = globalParameters["WorkingPath"] - assert wkpath == str(buildPath), f"path={wkpath} != buildPath={buildPath}" - # deleteme + buildPath = ensurePath(outputPath / "build") runScriptName = writeRunScript(buildPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, buildPath, configPaths) with ClientExecutionLock(): @@ -244,10 +201,6 @@ def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: if process.returncode: printWarning("ClientWriter Benchmark Process exited with code %u" % process.returncode) - # deleteme - popWorkingPath() # build - # deleteme - return process.returncode def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): @@ -297,10 +250,6 @@ def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cC if configPaths is None: configPaths = [] - # deleteme - assert str(buildDir) == globalParameters["WorkingPath"], f"buildDir={buildDir} != WP={globalParameters['WorkingPath']}" - # deleteme - configPaths.append(os.path.join(buildDir, "../source/ClientParameters.ini")) if enableTileSelection is True: configPaths.append(os.path.join(buildDir, "../source/ClientParameters_Granularity.ini")) @@ -714,10 +663,6 @@ def writeClientConfig( libraryFile = None ): - # deleteme - assert str(stepBaseDir / "source") == globalParameters["WorkingPath"], f"stepBaseDir={stepBaseDir} != WP={globalParameters['WorkingPath']}" - # deleteme - sourceDir = os.path.join(stepBaseDir, "source") if tileAwareSelection: diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 5b7e1c128c..9e80e487db 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -53,7 +53,6 @@ # Global Parameters ################################################################################ globalParameters = OrderedDict() -workingDirectoryStack = [] ######################################## # common @@ -241,9 +240,6 @@ globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi -# deleteme -globalParameters["WorkingPath"] = os.getcwd() # path where tensile called from -# deleteme globalParameters["IndexChars"] = "IJKLMNOPQRSTUVWXYZ" # which characters to use for C[ij]=Sum[k] A[ik]*B[jk] globalParameters["ScriptPath"] = os.path.dirname(os.path.realpath(__file__)) # path to Tensile/Tensile.py globalParameters["SourcePath"] = os.path.join(globalParameters["ScriptPath"], "Source") # path to Tensile/Source/ @@ -1790,29 +1786,6 @@ def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ else: destinationDictionary[key] = deepcopy(defaultDictionary[key]) -# deleteme -################################################################################ -# Push / Pop Working Path -# store a WorkingPath where to write files (like benchmark files) -################################################################################ -def pushWorkingPath( foldername ): - # Warning: this is not thread-safe, modifies the global WorkingPath! - globalParameters["WorkingPath"] = \ - os.path.join(globalParameters["WorkingPath"], foldername ) - return ensurePath( globalParameters["WorkingPath"] ) -def popWorkingPath(): - # Warning: this is not thread-safe, modifies the global WorkingPath! - if len(workingDirectoryStack) == 0: - globalParameters["WorkingPath"] = \ - os.path.split(globalParameters["WorkingPath"])[0] - else: - globalParameters["WorkingPath"] = workingDirectoryStack.pop() -def setWorkingPath( fullPathName ): - # Warning: this is not thread-safe, modifies the global WorkingPath! - workingDirectoryStack.append(globalParameters["WorkingPath"]) - globalParameters["WorkingPath"] = ensurePath(fullPathName) -# deleteme - def ensurePath(path): try: diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 78ec4e97ff..11cdfeedbf 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -24,7 +24,7 @@ from pathlib import Path from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ - setWorkingPath, popWorkingPath, assignParameterWithDefault, startTime, ProgressBar, printWarning + assignParameterWithDefault, startTime, ProgressBar, printWarning from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -1455,11 +1455,6 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) print2("# LibraryLogic config: %s" % config) print2("# DefaultAnalysisParameters: " % defaultAnalysisParameters) - # deleteme - setWorkingPath(libraryLogicPath) - assert libraryLogicPath == globalParameters["WorkingPath"], f"Lib logic path: {libraryLogicPath} benchmark data path: {benchmarkDataPath}" - # deleteme - # Assign Defaults analysisParameters = {} for parameter in defaultAnalysisParameters: @@ -1526,13 +1521,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) elapsedTime = currentTime - startTime print1("%s\n# Finish Analysing data to %s in %.3fs\n%s" % (HR, os.path.split(libraryLogicPath)[0], elapsedTime, HR) ) - # deleteme - popWorkingPath() - # deleteme -############################################################################## -# Error handling for frequency issues -############################################################################## def handle_frequency_issue(message): print1(message) print1(" - Type 'yes(y)' to abort the operation.") @@ -1558,25 +1547,6 @@ def handle_frequency_issue(message): ################################################################################ ################################################################################ def main(config, cxxCompiler: str, outputPath: Path): - benchmarkDataPath = outputPath / globalParameters["BenchmarkDataPath"] - - # deleteme - benchmarkDataPathOld = os.path.join(globalParameters["WorkingPath"], \ - globalParameters["BenchmarkDataPath"]) - assert benchmarkDataPathOld == str(benchmarkDataPath), f"benchmarkDataPathOld: {benchmarkDataPathOld}, benchmarkDataPath: {benchmarkDataPath}" - # deleteme - libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] - - # deleteme - libraryLogicPathOld = os.path.join(globalParameters["WorkingPath"], \ - globalParameters["LibraryLogicPath"]) - assert libraryLogicPathOld == str(libraryLogicPath), f"libraryLogicPathOld: {libraryLogicPathOld}, libraryLogicPath: {libraryLogicPath}" - # deleteme - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) - - # deleteme - assert globalParameters["WorkingPath"] == str(outputPath), f"WorkingPath: {globalParameters['WorkingPath']}, outputPath: {outputPath}" - # deleteme diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index a335b9fb61..b6396fab0d 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -303,6 +303,7 @@ def Tensile(userArgs): config["UseCache"] = useCache globalParameters["ConfigPath"] = configPaths + globalParameters["OutputPath"] = outputPath cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) @@ -311,13 +312,6 @@ def Tensile(userArgs): asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) - globalParameters["OutputPath"] = outputPath - - # deleteme - globalParameters["WorkingPath"] = outputPath - print("WorkingPath: %s" % globalParameters["WorkingPath"]) - # deleteme - overrideParameters = argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 945308d959..4697b891bf 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -38,8 +38,7 @@ from Tensile.Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults from Tensile.TensileInstructions import getGfxName, TensileInstructions from Tensile.Common import globalParameters, HR, print1, print2, printExit, ensurePath, \ - CHeader, assignGlobalParameters, architectureMap, IsaVersion, pushWorkingPath, \ - popWorkingPath, ParallelMap2 + CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2 from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H from Tensile import LibraryIO diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py index 4d9a51d8e9..b0af4f8fa5 100644 --- a/tensilelite/Tensile/TensileRetuneLibrary.py +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -29,18 +29,44 @@ from . import LibraryLogic from . import Common from .Common import globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ - pushWorkingPath, popWorkingPath, restoreDefaultGlobalParameters, HR + restoreDefaultGlobalParameters, HR from .Tensile import addCommonArguments, argUpdatedGlobalParameters from .SolutionStructs import ProblemSizes from .Toolchain.Validators import validateToolchain from . import __version__ +from pathlib import Path + import argparse import copy import os import shutil import sys +workingDirectoryStack = [] +def pushWorkingPath( foldername ): + # Warning: this is not thread-safe, modifies the global WorkingPath! + globalParameters["WorkingPath"] = \ + os.path.join(globalParameters["WorkingPath"], foldername ) + return ensurePath( globalParameters["WorkingPath"] ) +def popWorkingPath(): + # Warning: this is not thread-safe, modifies the global WorkingPath! + if len(workingDirectoryStack) == 0: + globalParameters["WorkingPath"] = \ + os.path.split(globalParameters["WorkingPath"])[0] + else: + globalParameters["WorkingPath"] = workingDirectoryStack.pop() +def ensurePath(path): + try: + os.makedirs(path) + except FileExistsError: + pass + return path +def setWorkingPath( fullPathName ): + # Warning: this is not thread-safe, modifies the global WorkingPath! + workingDirectoryStack.append(globalParameters["WorkingPath"]) + globalParameters["WorkingPath"] = ensurePath(fullPathName) + def parseCurrentLibrary(libPath, sizePath): libYaml = LibraryIO.read(libPath) @@ -141,6 +167,8 @@ def TensileRetuneLibrary(userArgs): libPath = args.LogicFile sizePath = args.SizeFile + libraryFormat = args.LibraryFormat + print1("# Library Logic: {}".format(libPath)) print1("#") print1(HR) @@ -161,14 +189,10 @@ def TensileRetuneLibrary(userArgs): ############################################## # Retuning ############################################## - outPath = ensurePath(os.path.abspath(args.OutputPath)) + outputPath = Path(ensurePath(os.path.abspath(args.OutputPath))) restoreDefaultGlobalParameters() - # deleteme -- "WorkingPath" - assignGlobalParameters({"LibraryFormat": "msgpack", - "OutputPath": outPath, - "WorkingPath": outPath}) - # deleteme + assignGlobalParameters({"LibraryFormat": libraryFormat, "OutputPath": outputPath}) overrideParameters = argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): @@ -177,26 +201,32 @@ def TensileRetuneLibrary(userArgs): # parse library logic then setup and run benchmarks (rawYaml, solutions, problemSizes) = parseCurrentLibrary(libPath, sizePath) - runBenchmarking(solutions, problemSizes, outPath, update, cxxCompiler, cCompiler, assembler, offloadBundler) + runBenchmarking(solutions, problemSizes, outputPath, update, cxxCompiler, cCompiler, assembler, offloadBundler) if remake: # write library logic file - LibraryLogic.main({"ScheduleName": rawYaml[1], - "ArchitectureName": rawYaml[2], - "DeviceNames": rawYaml[3] }) + LibraryLogic.main( + { + "ScheduleName": rawYaml[1], + "ArchitectureName": rawYaml[2], + "DeviceNames": rawYaml[3] + }, + cxxCompiler, + outputPath + ) if update: # read update yaml from benchmark client and update logic print1("") print1(HR) print1("# Reading update file from Benchmarking Client") - updateFile = os.path.join(outPath, "Data", "update.yaml") + updateFile = os.path.join(outputPath, "Data", "update.yaml") updateLogic = LibraryIO.read(updateFile) rawYaml[7] = updateLogic # write updated library logic (does not overwrite original) libName = os.path.basename(libPath) - outFile = os.path.join(outPath, libName) + outFile = os.path.join(outputPath, libName) print1("# Writing updated Library Logic: {}".format(outFile)) LibraryIO.writeYAML(outFile, rawYaml, explicit_start=False, explicit_end=False) From c2e09c61c15c34e1b4131ccd8804235aaa515ed5 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:06:14 +0000 Subject: [PATCH 007/105] fix: missing global parameter warning --- tensilelite/Tensile/Common.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 9e80e487db..9b4fcf2da6 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -1758,7 +1758,22 @@ def assignGlobalParameters(config, cxxCompiler=None): except (subprocess.CalledProcessError, OSError) as e: printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) + # The following keys may be present in the config, but are not (or no longer) global parameters. + ignoreKeys = [ + "UseCompression", + "CxxCompiler", + "CCompiler", + "OffloadBundler", + "Assembler", + "LogicPath", + "LogicFilter", + "OutputPath", + "Experimental", + "GenSolTable" + ] for key in config: + if key in ignoreKeys: + continue value = config[key] if key not in globalParameters: printWarning("Global parameter %s = %s unrecognised." % ( key, value )) From d3a11bc03429c3f5e91e47c9a08477268b87ca61 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:46:03 -0600 Subject: [PATCH 008/105] fix: ensure library logic path is built --- tensilelite/Tensile/LibraryLogic.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 11cdfeedbf..3606ac0523 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -24,7 +24,7 @@ from pathlib import Path from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ - assignParameterWithDefault, startTime, ProgressBar, printWarning + assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -1452,6 +1452,8 @@ def indicesToSerial(self, solutionIdx, indices ): def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str): + libraryLogicPath = ensurePath(libraryLogicPath) + print2("# LibraryLogic config: %s" % config) print2("# DefaultAnalysisParameters: " % defaultAnalysisParameters) @@ -1465,7 +1467,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) print1(HR) currentTime = time.time() elapsedTime = currentTime - startTime - print1("# Analysing data in %s - %.3fs" % (globalParameters["BenchmarkDataPath"], elapsedTime) ) + print1("# Analysing data in %s - %.3fs" % (benchmarkDataPath, elapsedTime) ) for parameter in analysisParameters: print2("# %s: %s" % (parameter, analysisParameters[parameter])) print1(HR) From 9517470f8dc5b72f66c3b3639908c02ba9fe3e8e Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:14:30 -0600 Subject: [PATCH 009/105] fix: client writer paths --- tensilelite/Tensile/ClientWriter.py | 4 ++-- tensilelite/Tensile/Tensile.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 22b3683e7d..becbdc166c 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -148,7 +148,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): activationArgs=activationArgs, icacheFlushArgs=icacheFlushArgs, stepName=str(ProblemType(problemType)), - stepBaseDir=str(sourcePath), + stepBaseDir=str(clientLibraryPath), newLibrary=newLibrary, configBase="ClientParameters_%s"%str(ProblemType(problemType)), codeObjectFiles=coList, @@ -309,7 +309,7 @@ def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cC runScriptFile.write("%s -d 0 --setfan 50\n" % globalParameters["ROCmSMIPath"]) else: for configFile in configPaths: - runScriptFile.write("{} --config-file {} {} --best-solution 1\n".format(ClientExecutable.getClientExecutable(cxxCompiler, cCompiler), configFile, globalParameters["ClientArgs"])) + runScriptFile.write("{} --config-file {} {} --best-solution 1\n".format(ClientExecutable.getClientExecutable(cxxCompiler, cCompiler, buildDir), configFile, globalParameters["ClientArgs"])) if os.name != "nt": runScriptFile.write("exit $ERR\n") runScriptFile.close() diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index b6396fab0d..40c7c4b651 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -153,13 +153,13 @@ def splitExtraParameters(par): help="set PrintLevel=2 and CMakeBuildType=Debug") argParser.add_argument("--short-names", dest="shortNames", action="store_true", \ help="use serial kernel and solution names") - argParser.add_argument("--cxx-compiler", dest="CxxCompiler", choices=[ToolchainDefaults.CXX_COMPILER], \ + argParser.add_argument("--cxx-compiler", dest="CxxCompiler", \ action="store", default=ToolchainDefaults.CXX_COMPILER, help="select which C++/HIP compiler to use") - argParser.add_argument("--c-compiler", dest="CCompiler", choices=[ToolchainDefaults.C_COMPILER], \ + argParser.add_argument("--c-compiler", dest="CCompiler", \ action="store", default=ToolchainDefaults.C_COMPILER, help="select which C compiler to use") - argParser.add_argument("--assembler", dest="Assembler", choices=[ToolchainDefaults.ASSEMBLER], \ + argParser.add_argument("--assembler", dest="Assembler", \ action="store", default=ToolchainDefaults.ASSEMBLER, help="select which assembler to use") - argParser.add_argument("--offload-bundler", dest="OffloadBundler", choices=[ToolchainDefaults.OFFLOAD_BUNDLER], \ + argParser.add_argument("--offload-bundler", dest="OffloadBundler", \ action="store", default=ToolchainDefaults.OFFLOAD_BUNDLER, help="select which offload bundler to use") argParser.add_argument("--logic-format", dest="LogicFormat", choices=["yaml", "json"], \ action="store", default="yaml", help="select which logic format to use") From 4f735985fc1557913d10e1d4c842539ae4f4e3ce Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 23 Jan 2025 16:38:24 -0600 Subject: [PATCH 010/105] fix: use child path for client libraries --- tensilelite/Tensile/ClientWriter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index becbdc166c..ea551aa8fb 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -170,7 +170,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): forBenchmark = False enableTileSelection = False - returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, outputPath, clientParametersPaths) + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, clientLibraryPath, clientParametersPaths) return returncode From df2868332e04570e94bdacdeefb3129d05638a0b Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:10:31 -0600 Subject: [PATCH 011/105] remove PrintTiming from global params --- tensilelite/Tensile/Common.py | 1 - tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 9b4fcf2da6..c7e41dfb33 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -60,7 +60,6 @@ globalParameters["MinimumRequiredVersion"] = "0.0.0" # which version of tensile is required to handle all the features required by this configuration file globalParameters["PerformanceMetric"] = "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} globalParameters["PrintLevel"] = 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose -globalParameters["PrintTiming"] = False # print duration for each stage in generator. globalParameters["ClientLogLevel"] = 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) # benchmarking globalParameters["KernelTime"] = False # T=use device timers, F=use host timers diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index 2ab382b480..5cbe71b440 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -71,8 +71,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: default=-1, help="Number of parallel jobs to launch.") argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, default=1, help="Set printout verbosity level.") - argParser.add_argument("--print-timing", dest="PrintTiming", - default=False, action="store_true", help="Print duration of each stage.") argParser.add_argument("--no-lazy-library-loading", dest="LazyLibraryLoading", action="store_false", default=True, help="Disable building for lazy library loading.") argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", @@ -108,7 +106,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: arguments["AMDGPUArchPath"] = False arguments["CpuThreads"] = args.CpuThreads arguments["PrintLevel"] = args.PrintLevel - arguments["PrintTiming"] = args.PrintTiming arguments["AsmDebug"] = args.AsmDebug arguments["BuildIdKind"] = args.BuildIdKind arguments["KeepBuildTmp"] = args.KeepBuildTmp From ea37c0667db68d25c7905eb199d2ef01f6524290 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:18:55 -0600 Subject: [PATCH 012/105] remove another assert --- tensilelite/Tensile/ClientExecutable.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index 35daaca0a7..d93c825514 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -59,10 +59,6 @@ def builtPath(self, path, *paths): def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cCompiler: str): sourcedir = globalParameters["SourcePath"] - builddirOld = os.path.join(globalParameters["OutputPath"], globalParameters["ClientBuildPath"]) - assert builddir is not None, "builddir must be provided" - assert builddirOld == str(builddir), f"builddir must be {builddirOld}, not {builddir}" - builddir = Common.ensurePath(builddir) options = {'CMAKE_BUILD_TYPE': globalParameters["CMakeBuildType"], From 8d72b2b0ea65183f01f05dc7ac3fd3274bfba50c Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 00:08:58 +0000 Subject: [PATCH 013/105] restore PrintTiming --- tensilelite/Tensile/Common.py | 1 + tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py | 3 +++ tensilelite/Tensile/TensileCreateLibrary/Run.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index c7e41dfb33..9b4fcf2da6 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -60,6 +60,7 @@ globalParameters["MinimumRequiredVersion"] = "0.0.0" # which version of tensile is required to handle all the features required by this configuration file globalParameters["PerformanceMetric"] = "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} globalParameters["PrintLevel"] = 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose +globalParameters["PrintTiming"] = False # print duration for each stage in generator. globalParameters["ClientLogLevel"] = 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) # benchmarking globalParameters["KernelTime"] = False # T=use device timers, F=use host timers diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index 5cbe71b440..2ab382b480 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -71,6 +71,8 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: default=-1, help="Number of parallel jobs to launch.") argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, default=1, help="Set printout verbosity level.") + argParser.add_argument("--print-timing", dest="PrintTiming", + default=False, action="store_true", help="Print duration of each stage.") argParser.add_argument("--no-lazy-library-loading", dest="LazyLibraryLoading", action="store_false", default=True, help="Disable building for lazy library loading.") argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", @@ -106,6 +108,7 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: arguments["AMDGPUArchPath"] = False arguments["CpuThreads"] = args.CpuThreads arguments["PrintLevel"] = args.PrintLevel + arguments["PrintTiming"] = args.PrintTiming arguments["AsmDebug"] = args.AsmDebug arguments["BuildIdKind"] = args.BuildIdKind arguments["KeepBuildTmp"] = args.KeepBuildTmp diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 4697b891bf..8da1c938c3 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -55,7 +55,7 @@ def wrapper(*args, **kwargs): res = func(*args, **kwargs) end = timer() - if globalParameters['PrintTiming']: + if globalParameters["PrintTiming"]: print(f'{func.__name__} took {end - start} seconds') return res From eab514525324b38f42d4dd54095e6621a51c8cfe Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:10:23 +0000 Subject: [PATCH 014/105] fix: reviewer comments --- tensilelite/Tensile/BenchmarkProblems.py | 3 +-- tensilelite/Tensile/Tensile.py | 9 --------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index e3f0fd54d3..04215ee647 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -117,9 +117,8 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, buildTmpPath: Path): """Write all the files needed for a given benchmarking step""" - ensurePath(sourcePath / "Solutions") - ensurePath(sourcePath / "Kernels") + ensurePath(sourcePath) copyStaticFiles(sourcePath) kernels = [] diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 40c7c4b651..4cf814a6e7 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -76,15 +76,7 @@ def executeStepsInConfig( cCompiler (str): The C compiler to use. """ - # Goal, define these here and pass them in to the relevant functions to it's clear - # what is being built and where buildTmpPath = outputPath / "build_tmp" - clientBuildPath = outputPath / "0_Build" - benchamrkProblemsPath = outputPath / "1_BenchmarkProblems" - benchmarkDataPath = outputPath / "2_BenchmarkData" - libraryLogicPath = outputPath / "3_LibraryLogic" - clientLibraryPath = outputPath / "4_LibraryClient" - ############################################################################## # Benchmark Problems ############################################################################## @@ -303,7 +295,6 @@ def Tensile(userArgs): config["UseCache"] = useCache globalParameters["ConfigPath"] = configPaths - globalParameters["OutputPath"] = outputPath cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) From 5599a46719aa3a3bd4bed816f1a9005ed51b83df Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 17:44:59 +0000 Subject: [PATCH 015/105] refactor: use env var for @timing decorator --- tensilelite/Tensile/Common.py | 1 - .../TensileCreateLibrary/ParseArguments.py | 3 -- .../Tensile/TensileCreateLibrary/Run.py | 15 +----- .../Utilities/{ => Decorators}/Profile.py | 12 +---- .../Tensile/Utilities/Decorators/Shared.py | 35 ++++++++++++++ .../Tensile/Utilities/Decorators/Timing.py | 46 +++++++++++++++++++ 6 files changed, 85 insertions(+), 27 deletions(-) rename tensilelite/Tensile/Utilities/{ => Decorators}/Profile.py (87%) create mode 100644 tensilelite/Tensile/Utilities/Decorators/Shared.py create mode 100644 tensilelite/Tensile/Utilities/Decorators/Timing.py diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 9b4fcf2da6..c7e41dfb33 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -60,7 +60,6 @@ globalParameters["MinimumRequiredVersion"] = "0.0.0" # which version of tensile is required to handle all the features required by this configuration file globalParameters["PerformanceMetric"] = "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} globalParameters["PrintLevel"] = 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose -globalParameters["PrintTiming"] = False # print duration for each stage in generator. globalParameters["ClientLogLevel"] = 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) # benchmarking globalParameters["KernelTime"] = False # T=use device timers, F=use host timers diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index 2ab382b480..5cbe71b440 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -71,8 +71,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: default=-1, help="Number of parallel jobs to launch.") argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, default=1, help="Set printout verbosity level.") - argParser.add_argument("--print-timing", dest="PrintTiming", - default=False, action="store_true", help="Print duration of each stage.") argParser.add_argument("--no-lazy-library-loading", dest="LazyLibraryLoading", action="store_false", default=True, help="Disable building for lazy library loading.") argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", @@ -108,7 +106,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: arguments["AMDGPUArchPath"] = False arguments["CpuThreads"] = args.CpuThreads arguments["PrintLevel"] = args.PrintLevel - arguments["PrintTiming"] = args.PrintTiming arguments["AsmDebug"] = args.AsmDebug arguments["BuildIdKind"] = args.BuildIdKind arguments["KeepBuildTmp"] = args.KeepBuildTmp diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 8da1c938c3..014473c79b 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -45,22 +45,11 @@ from Tensile.SolutionLibrary import MasterSolutionLibrary from Tensile.SolutionStructs import Solution from Tensile.CustomYamlLoader import load_logic_gfx_arch -from Tensile.Utilities.Profile import profile +from Tensile.Utilities.Decorators.Profile import profile +from Tensile.Utilities.Decorators.Timing import timing from .ParseArguments import parseArguments -def timing(func): - def wrapper(*args, **kwargs): - start = timer() - res = func(*args, **kwargs) - end = timer() - - if globalParameters["PrintTiming"]: - print(f'{func.__name__} took {end - start} seconds') - - return res - return wrapper - class KernelCodeGenResult(NamedTuple): err: int diff --git a/tensilelite/Tensile/Utilities/Profile.py b/tensilelite/Tensile/Utilities/Decorators/Profile.py similarity index 87% rename from tensilelite/Tensile/Utilities/Profile.py rename to tensilelite/Tensile/Utilities/Decorators/Profile.py index cc3c7eb44c..5439e2a8c5 100644 --- a/tensilelite/Tensile/Utilities/Profile.py +++ b/tensilelite/Tensile/Utilities/Decorators/Profile.py @@ -24,12 +24,13 @@ import cProfile import pstats -import os from pathlib import Path from datetime import datetime, timezone from typing import Callable, Tuple +from .Shared import envVariableIsSet + PROFILE_ENV_VAR = "TENSILE_PROFILE" def profile(func: Callable) -> Callable: @@ -52,15 +53,6 @@ def wrapper(*args, **kwargs): return output return wrapper -def envVariableIsSet(varName: str) -> bool: - """Checks if the provided environment variable is set to "ON", "TRUE", or "1" - Args: - varName: Environment variable name. - Returns: - True if the environment variable is set, otherwise False. - """ - value = os.environ.get(varName, "").upper() - return True if value in ["ON", "TRUE", "1"] else False def initProfileArtifacts(funcName: str) -> Tuple[Path, str]: """Initializes filenames and paths for profiling artifacts based on the current datetime diff --git a/tensilelite/Tensile/Utilities/Decorators/Shared.py b/tensilelite/Tensile/Utilities/Decorators/Shared.py new file mode 100644 index 0000000000..b87f0fbfc8 --- /dev/null +++ b/tensilelite/Tensile/Utilities/Decorators/Shared.py @@ -0,0 +1,35 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import os + +def envVariableIsSet(varName: str) -> bool: + """Checks if the provided environment variable is set to "YES", "ON", "TRUE", or "1" + Args: + varName: Environment variable name. + Returns: + True if the environment variable is set, otherwise False. + """ + value = os.environ.get(varName, "").upper() + return True if value in ["YES", "ON", "TRUE", "1"] else False \ No newline at end of file diff --git a/tensilelite/Tensile/Utilities/Decorators/Timing.py b/tensilelite/Tensile/Utilities/Decorators/Timing.py new file mode 100644 index 0000000000..475199b0b6 --- /dev/null +++ b/tensilelite/Tensile/Utilities/Decorators/Timing.py @@ -0,0 +1,46 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from typing import Callable +from timeit import default_timer as timer + +from .Shared import envVariableIsSet + +TIMING_ENV_VAR: str = "TENSILE_PRINT_TIMING" + +def timing(func: Callable) -> Callable: + f"""Timing decorator to measure execution time of a function. + + Add ``@timing`` to mark a function for timing; set the environment variable + {TIMING_ENV_VAR}=ON to enable timing decorated functions. + """ + if not envVariableIsSet(TIMING_ENV_VAR): + return func + def wrapper(*args, **kwargs): + start = timer() + res = func(*args, **kwargs) + end = timer() + print(f'{func.__name__} took {end - start} seconds') + return res + return wrapper \ No newline at end of file From 54072cc23658027c2cba27a95193eaa4dcfbb390 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 18:52:29 +0000 Subject: [PATCH 016/105] refactor: remove library print debug --- tensilelite/Tensile/Common.py | 1 - tensilelite/Tensile/GenerateSummations.py | 2 +- tensilelite/Tensile/Source/TensileCreateLibrary.cmake | 6 ------ 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index c7e41dfb33..7051e8a930 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -182,7 +182,6 @@ globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake globalParameters["CMakeCFlags"] = "" # pass flags to cmake globalParameters["DebugKernel"] = False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed -globalParameters["LibraryPrintDebug"] = False # solutions will print enqueue info when enqueueing a kernel globalParameters["AsanBuild"] = False # build with asan globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index df08fca4c6..a73ac3afe2 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -63,7 +63,7 @@ def createLibraryForBenchmark(logicPath, libraryPath, currentPath): pythonExePath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin", "TensileCreateLibrary") args = [pythonExePath, \ - "--new-client-only", "--no-short-file-names", "--no-library-print-debug", \ + "--new-client-only", "--no-short-file-names", \ "--architecture=all", "--code-object-version=default", "--library-format=yaml", \ logicPath, libraryPath, "HIP"] diff --git a/tensilelite/Tensile/Source/TensileCreateLibrary.cmake b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake index 168188255a..33c3df2565 100644 --- a/tensilelite/Tensile/Source/TensileCreateLibrary.cmake +++ b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake @@ -93,12 +93,6 @@ function(TensileCreateLibraryCmake set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--no-short-file-names") endif() - if(${Tensile_LIBRARY_PRINT_DEBUG}) - set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--library-print-debug") - else() - set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--no-library-print-debug") - endif() - set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--architecture=${Tensile_ARCHITECTURE}") set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--code-object-version=${Tensile_CODE_OBJECT_VERSION}") set(Tensile_CREATE_COMMAND ${Tensile_CREATE_COMMAND} "--cxx-compiler=${Tensile_COMPILER}") From 4265e9c888b93d1a551bcf27f9d9ee0d8d9d231f Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 19:07:43 +0000 Subject: [PATCH 017/105] refactor: remove exit after kernel gen option --- tensilelite/Tensile/Common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 7051e8a930..ee90c460a1 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -104,7 +104,6 @@ globalParameters["ShowProgressBar"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run globalParameters["SolutionSelectionAlg"] = 1 # algorithm to determine which solutions to keep. 0=removeLeastImportantSolutions, 1=keepWinnerSolutions (faster) globalParameters["ExpandRanges"] = True # expand ranges into exact configs before writing logic file. False ignores ranges. -globalParameters["ExitAfterKernelGen"] = False # Exit after generating kernels globalParameters["GenerateSourcesAndExit"] = False # Exit after kernel source generation. globalParameters["WavefrontWidth"] = 64 # if False and library client already built, then building library client will be skipped when tensile is re-run globalParameters["ExitOnFails"] = 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. From 08348e125e3c3b31e4db8bef9254b22a810d42ab Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 19:12:18 +0000 Subject: [PATCH 018/105] style: remove commented code --- tensilelite/Tensile/Common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index ee90c460a1..1b26c662ba 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -109,8 +109,6 @@ globalParameters["ExitOnFails"] = 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. globalParameters["CpuThreads"] = -1 # How many CPU threads to use for kernel generation. 0=no threading, -1 == nproc, N=min(nproc,N). TODO - 0 sometimes fails with a kernel name error? 0 does not check error codes correctly globalParameters["NumWarmups"] = 0 -# FROM MERGE -#globalParameters["CpuThreads"] = -4 # How many CPU threads to use for kernel generation. 0=no threading, <0 == nproc*abs(CpuThreads), N=min(nproc,N) # even if error occurs in kernel generation (ie due to resource overflow), # generate the kernel source anyway. Tensile will also attempt to run From 8c7b0a3f3bb1f658baaa653e4b63797d1b20ad4f Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:18:58 +0000 Subject: [PATCH 019/105] refactor: move common to a module --- tensilelite/Tensile/Activation.py | 2 +- tensilelite/Tensile/AsmAddressCalculation.py | 5 +- tensilelite/Tensile/AsmMemoryInstruction.py | 2 +- tensilelite/Tensile/AsmStoreState.py | 2 +- tensilelite/Tensile/BenchmarkProblems.py | 10 +- tensilelite/Tensile/ClientExecutable.py | 6 +- tensilelite/Tensile/ClientWriter.py | 19 +- tensilelite/Tensile/{ => Common}/Common.py | 176 ++++++++++-------- tensilelite/Tensile/Common/Constants.py | 64 +++++++ tensilelite/Tensile/{ => Common}/Parallel.py | 16 +- tensilelite/Tensile/Common/Types.py | 3 + tensilelite/Tensile/Common/__init__.py | 7 + .../Tensile/Components/ComputeStoreVgprs.py | 2 +- .../Tensile/Components/GlobalWriteBatch.py | 3 +- tensilelite/Tensile/Components/Signature.py | 3 +- tensilelite/Tensile/Contractions.py | 6 +- tensilelite/Tensile/EmbeddedData.py | 2 +- tensilelite/Tensile/KernelWriter.py | 11 +- .../Tensile/KernelWriterActivationOnly.py | 8 +- tensilelite/Tensile/KernelWriterAssembly.py | 37 ++-- tensilelite/Tensile/KernelWriterBetaOnly.py | 8 +- tensilelite/Tensile/KernelWriterConversion.py | 9 +- tensilelite/Tensile/KernelWriterReduction.py | 13 +- tensilelite/Tensile/LibraryIO.py | 5 +- tensilelite/Tensile/Ops/AMaxGenerator.py | 2 +- tensilelite/Tensile/Ops/LayerNormGenerator.py | 2 +- tensilelite/Tensile/Ops/SoftmaxGenerator.py | 2 +- tensilelite/Tensile/Properties.py | 2 +- tensilelite/Tensile/SolutionLibrary.py | 2 +- tensilelite/Tensile/SolutionStructs.py | 19 +- tensilelite/Tensile/Tensile.py | 4 +- tensilelite/Tensile/TensileClientConfig.py | 6 +- .../TensileCreateLibrary/ParseArguments.py | 4 +- .../Tensile/TensileCreateLibrary/Run.py | 17 +- .../TensileInstructions/Instructions.py | 1 - tensilelite/Tensile/TensileRetuneLibrary.py | 9 +- tensilelite/Tensile/TensileUpdateLibrary.py | 6 +- tensilelite/Tensile/Utils.py | 126 ------------- tensilelite/Tensile/__init__.py | 13 +- .../Tensile/cmake/TensileConfigVersion.cmake | 2 +- tensilelite/bump-version.sh | 2 +- tensilelite/tox.ini | 1 - 42 files changed, 297 insertions(+), 342 deletions(-) rename tensilelite/Tensile/{ => Common}/Common.py (96%) create mode 100644 tensilelite/Tensile/Common/Constants.py rename tensilelite/Tensile/{ => Common}/Parallel.py (95%) create mode 100644 tensilelite/Tensile/Common/Types.py create mode 100644 tensilelite/Tensile/Common/__init__.py delete mode 100644 tensilelite/Tensile/Utils.py diff --git a/tensilelite/Tensile/Activation.py b/tensilelite/Tensile/Activation.py index 31fd14b8da..83f031586e 100644 --- a/tensilelite/Tensile/Activation.py +++ b/tensilelite/Tensile/Activation.py @@ -31,7 +31,7 @@ TensileInstructions from .TensileInstructions.Enums import * from .TensileInstructions.Instructions import * -from .Common import printExit, printWarning, globalParameters +from .Common import printExit, printWarning from dataclasses import dataclass, field diff --git a/tensilelite/Tensile/AsmAddressCalculation.py b/tensilelite/Tensile/AsmAddressCalculation.py index 8d79c672b9..e0988103b8 100644 --- a/tensilelite/Tensile/AsmAddressCalculation.py +++ b/tensilelite/Tensile/AsmAddressCalculation.py @@ -22,8 +22,7 @@ from .TensileInstructions import Module, EXEC, vgpr, sgpr, log2, Label from .TensileInstructions.Instructions import * -from .Common import globalParameters -from .Utils import DataDirection +from .Common import INDEX_CHARS, DataDirection ############################################################################## # Fields associated with computing address @@ -192,7 +191,7 @@ def emitExtractAndScalePackedDims(self, kernel, ss, tmpVgpr, storeChar): # - tmp+0 may be the incoming packed coordinate 0, used on replay too # - tmp+1 is DIV output # - tmp+2 is scratch - idxChar= globalParameters["IndexChars"][idx] + idxChar= INDEX_CHARS[idx] module.addComment0("extract %s"%kw.sizeRef(idx)) assert(tmpVgpr+1 != packedBits) # bad since we still need packedBits below for remainder (can't overwrite here) module.add(MacroInstruction("V_MAGIC_DIV", \ diff --git a/tensilelite/Tensile/AsmMemoryInstruction.py b/tensilelite/Tensile/AsmMemoryInstruction.py index b840aae5db..cbad624864 100644 --- a/tensilelite/Tensile/AsmMemoryInstruction.py +++ b/tensilelite/Tensile/AsmMemoryInstruction.py @@ -22,11 +22,11 @@ # ################################################################################ -from .Common import printExit from .TensileInstructions import DSStoreB8, DSStoreB8HID16, ReadWriteInstruction, \ DSLoadD16HIU16, DSLoadD16HIU8, \ DSLoadU8, DSLoadU16, DSStoreD16HIB16, \ DSStoreB16 +from .Common import printExit from dataclasses import dataclass, field from typing import Type diff --git a/tensilelite/Tensile/AsmStoreState.py b/tensilelite/Tensile/AsmStoreState.py index f9cac19f9b..edf7a1cb9e 100644 --- a/tensilelite/Tensile/AsmStoreState.py +++ b/tensilelite/Tensile/AsmStoreState.py @@ -22,7 +22,7 @@ from .TensileInstructions import DataType from .AsmAddressCalculation import AddrCalculation -from .Utils import DataDirection +from .Common import DataDirection from math import ceil, trunc, modf from dataclasses import dataclass, field diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 04215ee647..20708b51bc 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -22,7 +22,6 @@ # ################################################################################ -import glob import os import shutil import sys @@ -34,18 +33,17 @@ from . import ClientExecutable from . import SolutionLibrary from . import LibraryIO -from . import Utils from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni -from .Common import globalParameters, HR, print1, print2, \ - printExit, printWarning, ensurePath, startTime, validParameters from .KernelWriterAssembly import KernelWriterAssembly from .SolutionStructs import Solution, ProblemType, ProblemSizes from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain +from .Common import globalParameters, HR, print1, print2, \ + printExit, printWarning, ensurePath, startTime, tqdm, state def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): @@ -127,7 +125,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelHelperNames = set() # get unique kernels and kernel helpers - for solution in Utils.tqdm(solutions, "Finding unique solutions"): + for solution in tqdm(solutions, "Finding unique solutions"): solutionKernels = solution.getKernels() for kernel in solutionKernels: kName = Solution.getKeyNoInternalArgs(kernel) @@ -159,7 +157,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, srcToolchain.compiler) newLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(newLibraryFile, Utils.state(newLibrary), globalParameters["LibraryFormat"]) + LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) codeObjectFiles = [os.path.relpath(f, sourcePath) \ for f in codeObjectFiles] diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index d93c825514..81f60b00e2 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -28,8 +28,8 @@ from typing import Optional from pathlib import Path -from . import Common -from .Common import globalParameters +from .Common import Common, globalParameters +from Tensile import SOURCE_PATH class CMakeEnvironment: def __init__(self, sourceDir, buildDir, **options): @@ -57,7 +57,7 @@ def builtPath(self, path, *paths): return os.path.join(self.buildDir, path, *paths) def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cCompiler: str): - sourcedir = globalParameters["SourcePath"] + sourcedir = SOURCE_PATH builddir = Common.ensurePath(builddir) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index ea551aa8fb..6c21bd72e9 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -22,21 +22,22 @@ # ################################################################################ -from pathlib import Path -from . import ClientExecutable -from . import LibraryIO -from .TensileInstructions import getGfxName, DataType -from .Common import globalParameters, ensurePath, print1, printExit, CHeader, printWarning, listToInitializer, ClientExecutionLock -from .SolutionStructs import Problem, ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs -from .TensileCreateLibrary import copyStaticFiles - import os import subprocess import shlex import shutil + +from pathlib import Path from enum import Enum from glob import glob +from . import ROOT_PATH +from . import ClientExecutable +from . import LibraryIO +from .TensileInstructions import getGfxName, DataType +from .Common import globalParameters, ensurePath, print1, printExit, CHeader, printWarning, listToInitializer, ClientExecutionLock +from .SolutionStructs import Problem, ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs +from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex from .Contractions import ProblemType as ContractionsProblemType @@ -207,7 +208,7 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): import io runScriptFile = io.StringIO() - callCreateLibraryCmd = globalParameters["ScriptPath"] + "/bin/TensileCreateLibrary" + callCreateLibraryCmd = ROOT_PATH + "/bin/TensileCreateLibrary" if not globalParameters["LazyLibraryLoading"]: callCreateLibraryCmd += " --no-lazy-library-loading" diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common/Common.py similarity index 96% rename from tensilelite/Tensile/Common.py rename to tensilelite/Tensile/Common/Common.py index 1b26c662ba..a224e1615d 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common/Common.py @@ -22,28 +22,23 @@ # ################################################################################ -from . import __version__ -from . import Parallel -from .TensileInstructions import getGfxName, TensileInstructions +from enum import Enum from collections import OrderedDict from copy import deepcopy -from typing import Tuple import math import os.path import subprocess import sys +import functools import time import re - -IsaVersion = Tuple[int, int, int] - +from Tensile.TensileInstructions import getGfxName, TensileInstructions +from Tensile import __version__ startTime = time.time() -ParallelMap = Parallel.ParallelMap -ParallelMap2 = Parallel.ParallelMap2 # print level # 0 - user wants no printing # 1 - user wants limited prints @@ -235,9 +230,6 @@ globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi -globalParameters["IndexChars"] = "IJKLMNOPQRSTUVWXYZ" # which characters to use for C[ij]=Sum[k] A[ik]*B[jk] -globalParameters["ScriptPath"] = os.path.dirname(os.path.realpath(__file__)) # path to Tensile/Tensile.py -globalParameters["SourcePath"] = os.path.join(globalParameters["ScriptPath"], "Source") # path to Tensile/Source/ globalParameters["HipClangVersion"] = "0.0.0" globalParameters["AMDClangVersion"] = "0.0.0" @@ -1882,6 +1874,105 @@ def printStatus(self): def finish(self): pass + +class DataDirection(Enum): + NONE = 0, + READ = 1, + WRITE = 2 + +class SpinnyThing: + def __init__(self): + self.chars = ['|', '/', '-', '\\'] + self.index = 0; + + def increment(self, value=1): + sys.stdout.write('\b' + self.chars[self.index]) + sys.stdout.flush() + self.index = (self.index + 1) % len(self.chars) + + def finish(self): + sys.stdout.write('\b*\n') + sys.stdout.flush() + +def iterate_progress(obj, *args, **kwargs): + try: + progress = ProgressBar(len(obj)) + except TypeError: + progress = SpinnyThing() + for o in obj: + yield o + progress.increment() + progress.finish() + +try: + from tqdm import tqdm +except ImportError: + tqdm = iterate_progress + +def state(obj): + if hasattr(obj, 'state'): + return obj.state() + + if hasattr(obj.__class__, 'StateKeys'): + rv = {} + for key in obj.__class__.StateKeys: + attr = key + if isinstance(key, tuple): + (key, attr) = key + rv[key] = state(getattr(obj, attr)) + return rv + + if isinstance(obj, dict): + return {k: state(v) for k, v in obj.items()} + + if isinstance(obj, (str, int, float)): + return obj + + try: + return [state(i) for i in obj] + except TypeError: + pass + + return obj + +def state_key_ordering(cls): + def tup(obj): + return tuple([getattr(obj, k) for k in cls.StateKeys]) + + def lt(a, b): + return tup(a) < tup(b) + def eq(a, b): + return tup(a) == tup(b) + + cls.__lt__ = lt + cls.__eq__ = eq + + return functools.total_ordering(cls) + +def hash_combine(*objs, **kwargs): + shift = 1 + if 'shift' in kwargs: + shift = kwargs['shift'] + + if len(objs) == 1: + objs = objs[0] + + rv = 0 + try: + it = iter(objs) + rv = next(it) + for value in it: + rv = (rv << shift) ^ value + except TypeError: + return objs + except StopIteration: + pass + return rv + +def hash_objs(*objs, **kwargs): + return hash(tuple(objs)) + + from copy import copy class Backup: """RAII class to restore backed up fields from object""" @@ -1895,64 +1986,3 @@ def __del__(self): for k, v in self.fields.items(): setattr(self.object, k, v) -# Append copyrights to all files generated by tensile since they belong to Tensile intellectual property -CMakeHeader = """################################################################################ -# -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- -# ies of the Software, and to permit persons to whom the Software is furnished -# to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- -# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- -# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -################################################################################ - -################################################### -# This file was generated by Tensile: # -# https://github.com/ROCmSoftwarePlatform/Tensile # -################################################### - - -""" - -CHeader = """/******************************************************************************* -* Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- -* ies of the Software, and to permit persons to whom the Software is furnished -* to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- -* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- -* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*******************************************************************************/ - -/************************************************** -* This file was generated by Tensile: * -* https://github.com/ROCmSoftwarePlatform/Tensile * -**************************************************/ - - -""" - -HR = "################################################################################" diff --git a/tensilelite/Tensile/Common/Constants.py b/tensilelite/Tensile/Common/Constants.py new file mode 100644 index 0000000000..3df7457b0a --- /dev/null +++ b/tensilelite/Tensile/Common/Constants.py @@ -0,0 +1,64 @@ +# Characters to use for GEMM naming, e.g., C[ij]=Sum[k] A[ik]*B[jk] +INDEX_CHARS: str = "IJKLMNOPQRSTUVWXYZ" + +HR = "################################################################################" + +# Append copyrights to all files generated by Tensile since they belong to Tensile intellectual property +CMakeHeader = """################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +################################################### +# This file was generated by Tensile: # +# https://github.com/ROCmSoftwarePlatform/Tensile # +################################################### + + +""" + +CHeader = """/******************************************************************************* +* Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +* ies of the Software, and to permit persons to whom the Software is furnished +* to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +* PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +* CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*******************************************************************************/ + +/************************************************** +* This file was generated by Tensile: * +* https://github.com/ROCm/Tensile * +**************************************************/ + + +""" diff --git a/tensilelite/Tensile/Parallel.py b/tensilelite/Tensile/Common/Parallel.py similarity index 95% rename from tensilelite/Tensile/Parallel.py rename to tensilelite/Tensile/Common/Parallel.py index 99950aba04..0c0abdf704 100644 --- a/tensilelite/Tensile/Parallel.py +++ b/tensilelite/Tensile/Common/Parallel.py @@ -30,6 +30,8 @@ from joblib import Parallel, delayed +from .Common import tqdm + def joblibParallelSupportsGenerator(): import joblib from packaging.version import Version @@ -50,11 +52,6 @@ def CPUThreadCount(enable=True): return min(cpu_count, 64) # Temporarily hack to fix oom issue, remove this after jenkin is fixed. return min(cpu_count, cpuThreads) -def OverwriteGlobalParameters(newGlobalParameters): - from . import Common - Common.globalParameters.clear() - Common.globalParameters.update(newGlobalParameters) - def pcallWithGlobalParamsMultiArg(f, args, newGlobalParameters): OverwriteGlobalParameters(newGlobalParameters) return f(*args) @@ -131,8 +128,7 @@ def ParallelMap(function, objects, message="", enable=True, method=None, maxTask mapFunc = None if mapFunc is not None: - from . import Utils - return list(mapFunc(function, Utils.tqdm(objects, message))) + return list(mapFunc(function, tqdm(objects, message))) mapFunc = pool.map if method: mapFunc = method(pool) @@ -159,14 +155,13 @@ def ParallelMap(function, objects, message="", enable=True, method=None, maxTask def ParallelMapReturnAsGenerator(function, objects, message="", enable=True, multiArg=True): from .Common import globalParameters - from . import Utils threadCount = CPUThreadCount(enable) print("{0}Launching {1} threads...".format(message, threadCount)) if threadCount <= 1 and globalParameters["ShowProgressBar"]: # Provide a progress bar for single-threaded operation. callFunc = lambda args: function(*args) if multiArg else lambda args: function(args) - return [callFunc(args) for args in Utils.tqdm(objects, message)] + return [callFunc(args) for args in tqdm(objects, message)] with concurrent.futures.ProcessPoolExecutor(max_workers=threadCount) as executor: resultFutures = (executor.submit(function, *arg if multiArg else arg) for arg in objects) @@ -186,12 +181,11 @@ def ParallelMap2(function, objects, message="", enable=True, multiArg=True, retu return ParallelMapReturnAsGenerator(function, objects, message, enable, multiArg) from .Common import globalParameters - from . import Utils threadCount = CPUThreadCount(enable) if threadCount <= 1 and globalParameters["ShowProgressBar"]: # Provide a progress bar for single-threaded operation. - return [function(*args) if multiArg else function(args) for args in Utils.tqdm(objects, message)] + return [function(*args) if multiArg else function(args) for args in tqdm(objects, message)] countMessage = "" try: diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py new file mode 100644 index 0000000000..5c887371f0 --- /dev/null +++ b/tensilelite/Tensile/Common/Types.py @@ -0,0 +1,3 @@ +from typing import Tuple + +IsaVersion = Tuple[int, int, int] \ No newline at end of file diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py new file mode 100644 index 0000000000..4d972028e4 --- /dev/null +++ b/tensilelite/Tensile/Common/__init__.py @@ -0,0 +1,7 @@ +from .Common import * +from .Constants import * +from .Parallel import * +from .Types import * + +# Dunder variables are not exported via `*` +from .Common import __version__ \ No newline at end of file diff --git a/tensilelite/Tensile/Components/ComputeStoreVgprs.py b/tensilelite/Tensile/Components/ComputeStoreVgprs.py index c937a8a71f..5209c34660 100644 --- a/tensilelite/Tensile/Components/ComputeStoreVgprs.py +++ b/tensilelite/Tensile/Components/ComputeStoreVgprs.py @@ -27,7 +27,7 @@ vectorStaticRemainder, RegisterPoolResource, vgpr, sgpr, log2, \ vectorStaticDivideAndRemainder from ..Component import ComputeStoreVgprs -from ..Utils import DataDirection +from ..Common import DataDirection class ComputeStoreVgprsVALU(ComputeStoreVgprs): kernel = {"EnableMatrixInstruction": False, diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index e5a11928b5..6bd50cb677 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -20,12 +20,11 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..Common import globalParameters +from ..Common import globalParameters, DataDirection from ..Component import GlobalWriteComponents from ..SolutionStructs import Solution from ..Activation import ActivationModule, ActivationType from ..AsmStoreState import StoreState -from ..Utils import DataDirection from ..TensileInstructions import Label, Module, EXEC, SDWAModifiers, VCC, SelectBit, \ vgpr, sgpr, replaceHolder, SaturateCastType, VCvtBF16toFP32, \ DataType, CvtType, RoundType diff --git a/tensilelite/Tensile/Components/Signature.py b/tensilelite/Tensile/Components/Signature.py index 89dc4d30d7..99dceb19f4 100644 --- a/tensilelite/Tensile/Components/Signature.py +++ b/tensilelite/Tensile/Components/Signature.py @@ -23,8 +23,7 @@ ################################################################################ from ..Component import Signature -from ..Common import globalParameters -from ..Utils import DataDirection +from ..Common import globalParameters, DataDirection from ..TensileInstructions import SignatureBase from ..TensileInstructions import SignatureValueKind as SVK from ..Activation import ActivationType diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index db07b5d062..04847817ae 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -23,16 +23,12 @@ ################################################################################ from .Activation import ActivationType -from .Common import internalParameters, printExit from .TensileInstructions import DataType from . import Hardware from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Utils import state, state_key_ordering - -from . import Common -from . Common import globalParameters +from .Common import Common, internalParameters, globalParameters, state, state_key_ordering @state_key_ordering class FreeIndex: diff --git a/tensilelite/Tensile/EmbeddedData.py b/tensilelite/Tensile/EmbeddedData.py index d2b1641d34..b61cca966b 100644 --- a/tensilelite/Tensile/EmbeddedData.py +++ b/tensilelite/Tensile/EmbeddedData.py @@ -22,7 +22,7 @@ # ################################################################################ -from . import Common +from .Common import Common import itertools import os diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index a9fcbcea01..70e13a8370 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -22,8 +22,7 @@ # ################################################################################ -from . import Common -from .TensileInstructions import Item, TensileInstructions, slash50, replaceHolder, \ +from .TensileInstructions import TensileInstructions, replaceHolder, \ KernelBody, Module, StructuredModule, TextBlock, Dump, LabelManager, \ RegisterPool, Assert, fastdeepcopy, TensileInstructionsPassOptions, \ TensileInstructionsPass, \ @@ -31,15 +30,13 @@ from .TensileInstructions.Instructions import * from .KernelWriterModules import * from .TensilePass import TensilePass, TensilePassOptions -from .Common import globalParameters, CHeader, print1, printWarning, roundUp, Backup, print2, printExit from .Component import Component, LraTileProperties from .Components.Signature import UserArgumentsInfo from .CustomKernels import isCustomKernelConfig from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction -from .Utils import DataDirection - from .Activation import ActivationModule +from .Common import globalParameters, printWarning, roundUp, print2, printExit, INDEX_CHARS, DataDirection import abc import os @@ -2945,8 +2942,8 @@ def initKernel(self, kernel, tensorParametersA, tensorParametersB): self.states.doShadowInit = 1 # 1 is just store setup self.states.indexChars = [] - for i in range(0, len(globalParameters["IndexChars"])): - self.states.indexChars.append(globalParameters["IndexChars"][i]) + for i in range(0, len(INDEX_CHARS)): + self.states.indexChars.append(INDEX_CHARS[i]) self.states.indexChars[kernel["ProblemType"]["Index0"]] \ = "0" + self.states.indexChars[kernel["ProblemType"]["Index0"]] self.states.indexChars[kernel["ProblemType"]["Index1"]] \ diff --git a/tensilelite/Tensile/KernelWriterActivationOnly.py b/tensilelite/Tensile/KernelWriterActivationOnly.py index 1d1053cb28..c4dea91e04 100644 --- a/tensilelite/Tensile/KernelWriterActivationOnly.py +++ b/tensilelite/Tensile/KernelWriterActivationOnly.py @@ -22,7 +22,7 @@ from copy import deepcopy -from .Common import globalParameters, CHeader +from .Common import INDEX_CHARS from .KernelWriterBase import KernelWriterBase class KernelWriterActivationOnly(KernelWriterBase): @@ -39,8 +39,8 @@ def __init__(self, state): # determine chars for fast access self.states.indexChars = [] - for i in range(0, len(globalParameters["IndexChars"])): - self.states.indexChars.append(globalParameters["IndexChars"][i]) + for i in range(0, len(INDEX_CHARS)): + self.states.indexChars.append(INDEX_CHARS[i]) self.states.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.states.indexChars[self.state["ProblemType"]["Index0"]] self.states.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.states.indexChars[self.state["ProblemType"]["Index1"]] self.tileChar0 = self.states.indexChars[self.state["ProblemType"]["Index0"]] @@ -209,7 +209,7 @@ def kernelBody(self): def getKernelName(self): - indexChars = globalParameters["IndexChars"] + indexChars = INDEX_CHARS # C dimensions name = "D" for i in range(0, self.state["ProblemType"]["NumIndicesC"]): diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 5b415b494a..15790ecdd9 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -38,7 +38,6 @@ LabelManager, Assert from .TensileInstructions.Instructions import * from .TensilePass import getActivationFunctionModuleName, getActivationBranchModuleName -from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath from .TensileInstructions.Containers import HWRegContainer from .Component import Component from .KernelWriter import KernelWriter, ConstValues, StateValues, StateVgprs, CodeModules @@ -47,8 +46,8 @@ from .AsmStoreState import StoreState, VectorDataTypes from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType -from .Utils import DataDirection from .CustomKernels import isCustomKernelConfig +from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection from math import ceil, log, floor from copy import deepcopy @@ -164,12 +163,12 @@ def sizeRef(self, idx): See above definitions for how these are mapped to Free or Sum sizes based on the problem definition. """ - idxChar= globalParameters["IndexChars"][idx] + idxChar= INDEX_CHARS[idx] return sgpr("Size%s"%idxChar) def loopChar(self, kernel, loopIdx): loopDim = kernel["ProblemType"]["IndicesSummation"][loopIdx] - return globalParameters["IndexChars"][loopDim] + return INDEX_CHARS[loopDim] def loopSizeRef(self, kernel, loopIdx): loopDim = kernel["ProblemType"]["IndicesSummation"][loopIdx] @@ -762,7 +761,7 @@ def macroAndSet(self, kernel, tPA, tPB) -> Module: module.addComment0("Size Assignments") problemType = kernel["ProblemType"] for idx in range(max(problemType["IndexAssignmentsA"] + problemType["IndexAssignmentsB"])+1): - idxChar= globalParameters["IndexChars"][idx] + idxChar= INDEX_CHARS[idx] if idx in problemType["IndicesFree"] or idx in problemType["IndicesBatch"]: idxType="Free" elif idx in problemType["IndicesSummation"]: @@ -931,13 +930,13 @@ def macroAndSet(self, kernel, tPA, tPB) -> Module: dest = "v[\\vgprTmp+0]" needAdd = 1 macro.add(VSubU32(dst=dest, \ - src0=sgpr("Size%s"%globalParameters["IndexChars"][indices[i]]), \ + src0=sgpr("Size%s"%INDEX_CHARS[indices[i]]), \ src1=1, \ - comment="mirror %s%s 1"%(tc, globalParameters["IndexChars"][indices[i]]))) + comment="mirror %s%s 1"%(tc, INDEX_CHARS[indices[i]]))) macro.add(VMulLOU32(dst=dest, \ src0=dest, \ src1=self.strideRef(tc, indices[i]), \ - comment="mirror %s%s 2"%(tc, globalParameters["IndexChars"][indices[i]]))) + comment="mirror %s%s 2"%(tc, INDEX_CHARS[indices[i]]))) if needAdd: writeDirectToAddr = 0 # safety net, once we write address can't directly overwrite it later @@ -1001,14 +1000,14 @@ def macroAndSet(self, kernel, tPA, tPB) -> Module: if isMirrorIdx: macro.add(VSubI32( dst="v[\\vgprTmp+0]", - src0=sgpr("Size%s"%globalParameters["IndexChars"][idx]), \ + src0=sgpr("Size%s"%INDEX_CHARS[idx]), \ src1=offset, \ - comment="mirror %s%s 1"%(tc, globalParameters["IndexChars"][indices[i]]))) + comment="mirror %s%s 1"%(tc, INDEX_CHARS[indices[i]]))) macro.add(VSubI32(\ dst="v[\\vgprTmp+0]", src0="v[\\vgprTmp+0]", \ src1=1, \ - comment="mirror %s%s 2"%(tc, globalParameters["IndexChars"][indices[i]]))) + comment="mirror %s%s 2"%(tc, INDEX_CHARS[indices[i]]))) offset = "v[\\vgprTmp+0]" # offset * stride @@ -2019,7 +2018,7 @@ def extractPackedCoord1ToRowStart(self, kernel, packedC1, packedCoordVgpr, store module.add(VMovB32(dst=vgpr(tmpV0), src=vgpr(packedCoordVgpr), comment="copy coord1 then unpack")) for i,idx in enumerate(packedC1[:-1]): - idxChar= globalParameters["IndexChars"][idx] + idxChar= INDEX_CHARS[idx] module.addComment0("extract %s"%self.sizeRef(idx)) module.add(MacroInstruction(name="V_MAGIC_DIV", \ args=[tmpV1, vgpr(tmpV0), sgpr("MagicNumberSize%s"%idxChar), \ @@ -2349,16 +2348,16 @@ def graTileOffsets(self, kernel, tP, margin=-1): module.addSpaceLine() for p in range(0, numExtraPackedOffsetsPerTile): groIdx = tP["PackedIndices"][p+1] - groChar = globalParameters["IndexChars"][tP["PackedIndices"][p+1]] + groChar = INDEX_CHARS[tP["PackedIndices"][p+1]] groVgpr = vgpr(tP["vgprPackedOffsets"] + l*numExtraPackedOffsetsPerTile + p) - pChar = globalParameters["IndexChars"][tP["PackedIndices"][p]] + pChar = INDEX_CHARS[tP["PackedIndices"][p]] module.add(MacroInstruction(name="V_MAGIC_DIV", \ args=[tmpV, lastGroVgpr, sgpr("MagicNumberSize%s"%pChar), \ sgpr("MagicShiftSize%s"%pChar), (sgpr("MagicAbitSize%s"%pChar) if kernel["MagicDivAlg"]==2 else "0")] )) module.add(VMovB32(dst=groVgpr, src=vgpr(tmpV), comment="extract gro%s%s_%u (%s)"%(tc,groChar,l,groVgpr))) module.add(VMulLOU32(dst=vgpr(tmpV), src0=groVgpr, src1=sgpr("SizesFree+%u"%lastGroIdx), comment="remainder part 1")) module.add(VSubU32(dst=lastGroVgpr, src0=lastGroVgpr, src1=vgpr(tmpV), \ - comment="remove extracted bits from gro%s%s_%u (%s)"%(tc, globalParameters["IndexChars"][lastGroIdx], l, lastGroVgpr))) + comment="remove extracted bits from gro%s%s_%u (%s)"%(tc, INDEX_CHARS[lastGroIdx], l, lastGroVgpr))) lastGroVgpr = groVgpr lastGroIdx = groIdx self.vgprPool.checkIn(tmpV) @@ -8111,15 +8110,15 @@ def computeStoreSrdStart(self, kernel, srdTcList: list, sgprBpeList = [], useSiz # Skip cause stride = 1 if use size instead continue if i > 1: - strideC0 = "Size%s"%(globalParameters["IndexChars"][0]) - strideC1 = "Size%s"%(globalParameters["IndexChars"][1]) + strideC0 = "Size%s"%(INDEX_CHARS[0]) + strideC1 = "Size%s"%(INDEX_CHARS[1]) module.add(SMulI32(dst=sgpr(tmpS0), src0=sgpr(strideC0), src1=sgpr(strideC1))) for x in range(2, i - 1): - strideC = "Size%s"%(globalParameters["IndexChars"][x]) + strideC = "Size%s"%(INDEX_CHARS[x]) module.add(SMulI32(dst=sgpr(tmpS0), src0=sgpr(tmpS0), src1=sgpr(strideC))) module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpS0), sgpr(tmpS1), coord, sgpr(tmpS0), "Scale%s %s by Stride"%(mat, coord))) else: - strideC = "Size%s"%(globalParameters["IndexChars"][i-1]) + strideC = "Size%s"%(INDEX_CHARS[i-1]) module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpS0), sgpr(tmpS1), coord, sgpr(strideC), "Scale%s %s by Stride"%(mat, coord))) else: strideC = "Stride%s%s"%(mat, self.states.indexChars[i]) diff --git a/tensilelite/Tensile/KernelWriterBetaOnly.py b/tensilelite/Tensile/KernelWriterBetaOnly.py index ccf7886c14..b7096f15d3 100644 --- a/tensilelite/Tensile/KernelWriterBetaOnly.py +++ b/tensilelite/Tensile/KernelWriterBetaOnly.py @@ -24,7 +24,7 @@ from copy import deepcopy -from .Common import globalParameters, CHeader +from .Common import globalParameters, CHeader, INDEX_CHARS from .TensileInstructions import DataType from .KernelWriterBase import KernelWriterBase @@ -43,8 +43,8 @@ def __init__(self, state): # determine chars for fast access self.indexChars = [] - for i in range(0, len(globalParameters["IndexChars"])): - self.indexChars.append(globalParameters["IndexChars"][i]) + for i in range(0, len(INDEX_CHARS)): + self.indexChars.append(INDEX_CHARS[i]) self.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.indexChars[self.state["ProblemType"]["Index0"]] self.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.indexChars[self.state["ProblemType"]["Index1"]] self.tileChar0 = self.indexChars[self.state["ProblemType"]["Index0"]] @@ -281,7 +281,7 @@ def kernelBodyBetaOnly(self): def getKernelName(self): - indexChars = globalParameters["IndexChars"] + indexChars = INDEX_CHARS # C dimensions name = "C" for i in range(0, self.state["ProblemType"]["NumIndicesC"]): diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 905d5ad625..5d3436f58a 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -24,10 +24,11 @@ from copy import deepcopy -from .Common import globalParameters, CHeader, gfxArch, getGfxName from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType +from .Common import globalParameters, gfxArch, getGfxName, INDEX_CHARS + class KernelWriterConversion(KernelWriterBase): def __init__(self, state, load_vw): @@ -59,8 +60,8 @@ def __init__(self, state, load_vw): # determine chars for fast access self.indexChars = [] - for i in range(0, len(globalParameters["IndexChars"])): - self.indexChars.append(globalParameters["IndexChars"][i]) + for i in range(0, len(INDEX_CHARS)): + self.indexChars.append(INDEX_CHARS[i]) self.indexChars[self.state["ProblemType"]["Index0"]] = "0" + self.indexChars[self.state["ProblemType"]["Index0"]] self.indexChars[self.state["ProblemType"]["Index1"]] = "1" + self.indexChars[self.state["ProblemType"]["Index1"]] self.tileChar0 = self.indexChars[self.state["ProblemType"]["Index0"]] @@ -778,7 +779,7 @@ def kernelBody(self): def getKernelName(self): - indexChars = globalParameters["IndexChars"] + indexChars = INDEX_CHARS # C dimensions name = "C" for i in range(0, self.state["ProblemType"]["NumIndicesC"]): diff --git a/tensilelite/Tensile/KernelWriterReduction.py b/tensilelite/Tensile/KernelWriterReduction.py index c24472b807..44fdf8a7e5 100644 --- a/tensilelite/Tensile/KernelWriterReduction.py +++ b/tensilelite/Tensile/KernelWriterReduction.py @@ -24,7 +24,7 @@ from copy import deepcopy -from .Common import globalParameters, CHeader +from .Common import INDEX_CHARS from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType @@ -35,12 +35,11 @@ def __init__(self, state): self.state["ProblemType"] = deepcopy(state["ProblemType"]) - indexChars = globalParameters["IndexChars"] # C dimensions self.indicesStr = "" for i in range(0, self.state["ProblemType"]["NumIndicesC"]): - c = indexChars[i].lower() - self.indicesStr += indexChars[i].lower() + c = INDEX_CHARS[i].lower() + self.indicesStr += INDEX_CHARS[i].lower() # derive parameter self.language = "HIP" @@ -65,14 +64,14 @@ def getKernelName(self): def getHeaderFileString(self): fileString = "" # CHeader - indexChars = globalParameters["IndexChars"] + # C dimensions indicesStr = "" for i in range(0, self.state["ProblemType"]["NumIndicesC"]): - c = indexChars[i].lower() + c = INDEX_CHARS[i].lower() if c == 'k': continue - indicesStr += indexChars[i].lower() + indicesStr += INDEX_CHARS[i].lower() computeStr = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) computeChar = self.state["ProblemType"]["ComputeDataType"].toChar() diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index fd5b5beafb..78e723dd49 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -22,18 +22,17 @@ # ################################################################################ -from .Common import printExit, printWarning, print2, versionIsCompatible from .CustomKernels import getCustomKernelConfig from .SolutionStructs import Solution, ProblemSizes, ProblemType -from . import __version__ -from . import Common from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream +from .Common import Common, printExit, printWarning, print2, versionIsCompatible, __version__ from typing import NamedTuple, List import os import sys + try: import orjson as json except ImportError: diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index a579db0f3d..97042c9671 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -32,7 +32,7 @@ import collections from contextlib import contextmanager import Tensile.TensileInstructions as ti -from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index 093a437544..e1180cce6f 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -32,7 +32,7 @@ import collections from contextlib import contextmanager import Tensile.TensileInstructions as ti -from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index 1078615d84..8b89663324 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -31,7 +31,7 @@ import subprocess from contextlib import contextmanager import Tensile.TensileInstructions as ti -from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/Properties.py b/tensilelite/Tensile/Properties.py index 7c06c0f988..2df74fd494 100644 --- a/tensilelite/Tensile/Properties.py +++ b/tensilelite/Tensile/Properties.py @@ -22,7 +22,7 @@ # ################################################################################ -from .Utils import hash_objs, state +from .Common import hash_objs, state class Property: diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index f23a04bcbf..c8138b77cf 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -29,7 +29,7 @@ from . import Common from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Utils import state +from .Common import state class SingleSolutionLibrary: Tag = "Single" diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 05e66a313c..cff2818ccd 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -22,13 +22,6 @@ # ################################################################################ -from .Common import assignParameterWithDefault, \ - defaultProblemType, defaultSolution, \ - defaultInternalSupportParams, \ - globalParameters, internalParameters, \ - print2, printExit, printWarning, \ - validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA from .TensileInstructions import DataType, roundUpToNearestMultiple from .TensileInstructions.Base import fastdeepcopy as deepcopy @@ -44,6 +37,14 @@ from .CustomKernels import isCustomKernelConfig +from .Common import assignParameterWithDefault, \ + defaultProblemType, defaultSolution, \ + defaultInternalSupportParams, \ + globalParameters, internalParameters, \ + print2, printExit, printWarning, \ + validMFMA, validSMFMA, validParameters, \ + validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS + from collections import OrderedDict from collections.abc import Mapping from enum import Enum @@ -458,7 +459,7 @@ def assignDerivedParameters(state): ######################################## def __str__(self): - indexChars = globalParameters["IndexChars"] + indexChars = INDEX_CHARS # C dimensions name = "C" for i in range(0, self["NumIndicesC"]): @@ -2409,7 +2410,7 @@ def assignDerivedParameters(state): # grid size [0,1] state["PackedC0IdxChars"] = [] state["PackedC0IndicesX"] = [] - indexChars = globalParameters["IndexChars"] + indexChars = INDEX_CHARS # Pack all the dimensions (free) of A into grid[0] if problemType["Index0"] in problemType["IndexAssignmentsA"]: diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 4cf814a6e7..a63915997e 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -26,11 +26,12 @@ print("This file can no longer be run as a script. Run 'Tensile/bin/Tensile' instead.") exit(1) +import joblib import os import sys import argparse from .Common import globalParameters, print1, printExit, printWarning, ensurePath, \ - assignGlobalParameters, restoreDefaultGlobalParameters, HR + assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__ from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain from .Toolchain.Validators import validateToolchain, ToolchainDefaults @@ -38,7 +39,6 @@ from . import ClientWriter from . import LibraryIO from . import LibraryLogic -from . import __version__ from datetime import datetime from pathlib import Path diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py index be9a2c14f8..5c40efd179 100644 --- a/tensilelite/Tensile/TensileClientConfig.py +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -22,15 +22,13 @@ # ################################################################################ -from . import Common from . import ClientWriter from . import LibraryIO from .Contractions import ProblemType as ContractionsProblemType from .SolutionStructs import ProblemSizes, ProblemType -from .Common import print1, printExit, printWarning, assignGlobalParameters, \ - restoreDefaultGlobalParameters, HR +from .Common import Common, print1, printExit, printWarning, assignGlobalParameters, \ + restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters -from . import __version__ import argparse import os diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index 5cbe71b440..9e50528f6c 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -22,13 +22,13 @@ # ################################################################################ -from Tensile.Toolchain.Validators import ToolchainDefaults - import os from argparse import ArgumentParser from typing import Any, Dict, List, Optional from Tensile.Common import architectureMap +from Tensile.Toolchain.Validators import ToolchainDefaults + def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: """Parse command line arguments for TensileCreateLibrary. diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 014473c79b..18846e5d5c 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -32,16 +32,15 @@ from timeit import default_timer as timer from typing import NamedTuple, List, Optional, Sequence, Union -from Tensile import Utils +from Tensile import LibraryIO, SOURCE_PATH from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles from Tensile.Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults from Tensile.TensileInstructions import getGfxName, TensileInstructions -from Tensile.Common import globalParameters, HR, print1, print2, printExit, ensurePath, \ - CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2 +from Tensile.Common import globalParameters, HR, print1, print2, printExit, IsaVersion, ensurePath, state, \ + CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2, tqdm from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H -from Tensile import LibraryIO from Tensile.SolutionLibrary import MasterSolutionLibrary from Tensile.SolutionStructs import Solution from Tensile.CustomYamlLoader import load_logic_gfx_arch @@ -82,7 +81,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, removeSolutions = [] removeResults = [] - for kernIdx, r in Utils.tqdm(enumerate(results)) if globalParameters["PrintLevel"] > 1 else enumerate(results): + for kernIdx, r in tqdm(enumerate(results)) if globalParameters["PrintLevel"] > 1 else enumerate(results): if r.err != 0: if not errorTolerant: print("\nKernel generation failed for kernel: {}".format(kernels[kernIdx]["SolutionIndex"])) @@ -99,7 +98,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, for kern in removeKernels: kernels.remove(kern) - for solution in Utils.tqdm(solutions, "Finding invalid solutions") if globalParameters["PrintLevel"] > 1 else solutions: + for solution in tqdm(solutions, "Finding invalid solutions") if globalParameters["PrintLevel"] > 1 else solutions: solutionKernels = solution.getKernels() for kernel in solutionKernels: kName = Solution.getKeyNoInternalArgs(kernel) @@ -263,7 +262,7 @@ def copyStaticFiles(outputPath): "memory_gfx.h" ] for fileName in libraryStaticFiles: - shutil.copy(os.path.join(globalParameters["SourcePath"], fileName), outputPath) + shutil.copy(os.path.join(SOURCE_PATH, fileName), outputPath) return libraryStaticFiles @@ -447,11 +446,11 @@ def validLogicFile(p: Path): else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_"+archName) newMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, Utils.state(newMasterLibrary), arguments["LibraryFormat"]) + LibraryIO.write(masterFile, state(newMasterLibrary), arguments["LibraryFormat"]) for name, lib in newMasterLibrary.lazyLibraries.items(): filename = os.path.join(newLibraryDir, name) lib.applyNaming(kernelMinNaming) - LibraryIO.write(filename, Utils.state(lib), arguments["LibraryFormat"]) + LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) print1("# Tensile Library Writer DONE") print1(HR) diff --git a/tensilelite/Tensile/TensileInstructions/Instructions.py b/tensilelite/Tensile/TensileInstructions/Instructions.py index 6ccf6d6271..2b1e8e8d0f 100644 --- a/tensilelite/Tensile/TensileInstructions/Instructions.py +++ b/tensilelite/Tensile/TensileInstructions/Instructions.py @@ -28,7 +28,6 @@ import abc from enum import Enum from typing import List, Optional, Union -#from .Utils import sgpr ################################################################################ ################################################################################ diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py index b0af4f8fa5..cf588e5c40 100644 --- a/tensilelite/Tensile/TensileRetuneLibrary.py +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -27,21 +27,18 @@ from . import ClientWriter from . import LibraryIO from . import LibraryLogic -from . import Common -from .Common import globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ - restoreDefaultGlobalParameters, HR +from .Common import Common, globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ + restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters from .SolutionStructs import ProblemSizes from .Toolchain.Validators import validateToolchain -from . import __version__ - -from pathlib import Path import argparse import copy import os import shutil import sys +from pathlib import Path workingDirectoryStack = [] def pushWorkingPath( foldername ): diff --git a/tensilelite/Tensile/TensileUpdateLibrary.py b/tensilelite/Tensile/TensileUpdateLibrary.py index cf1685292c..5ff265d0ed 100644 --- a/tensilelite/Tensile/TensileUpdateLibrary.py +++ b/tensilelite/Tensile/TensileUpdateLibrary.py @@ -23,10 +23,10 @@ ################################################################################ from . import LibraryIO -from .Common import assignGlobalParameters, print1, restoreDefaultGlobalParameters, HR, \ - globalParameters, architectureMap, ensurePath, ParallelMap from .Tensile import addCommonArguments, argUpdatedGlobalParameters -from . import __version__ + +from .Common import assignGlobalParameters, print1, restoreDefaultGlobalParameters, HR, \ + globalParameters, architectureMap, ensurePath, ParallelMap, __version__ import argparse import copy diff --git a/tensilelite/Tensile/Utils.py b/tensilelite/Tensile/Utils.py deleted file mode 100644 index 8e8d6c6321..0000000000 --- a/tensilelite/Tensile/Utils.py +++ /dev/null @@ -1,126 +0,0 @@ -################################################################################ -# -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -from enum import Enum -from .Common import ProgressBar - -import functools -import sys - -class DataDirection(Enum): - NONE = 0, - READ = 1, - WRITE = 2 - -class SpinnyThing: - def __init__(self): - self.chars = ['|', '/', '-', '\\'] - self.index = 0; - - def increment(self, value=1): - sys.stdout.write('\b' + self.chars[self.index]) - sys.stdout.flush() - self.index = (self.index + 1) % len(self.chars) - - def finish(self): - sys.stdout.write('\b*\n') - sys.stdout.flush() - -def iterate_progress(obj, *args, **kwargs): - try: - progress = ProgressBar(len(obj)) - except TypeError: - progress = SpinnyThing() - for o in obj: - yield o - progress.increment() - progress.finish() - -try: - from tqdm import tqdm -except ImportError: - tqdm = iterate_progress - -def state(obj): - if hasattr(obj, 'state'): - return obj.state() - - if hasattr(obj.__class__, 'StateKeys'): - rv = {} - for key in obj.__class__.StateKeys: - attr = key - if isinstance(key, tuple): - (key, attr) = key - rv[key] = state(getattr(obj, attr)) - return rv - - if isinstance(obj, dict): - return {k: state(v) for k, v in obj.items()} - - if isinstance(obj, (str, int, float)): - return obj - - try: - return [state(i) for i in obj] - except TypeError: - pass - - return obj - -def state_key_ordering(cls): - def tup(obj): - return tuple([getattr(obj, k) for k in cls.StateKeys]) - - def lt(a, b): - return tup(a) < tup(b) - def eq(a, b): - return tup(a) == tup(b) - - cls.__lt__ = lt - cls.__eq__ = eq - - return functools.total_ordering(cls) - -def hash_combine(*objs, **kwargs): - shift = 1 - if 'shift' in kwargs: - shift = kwargs['shift'] - - if len(objs) == 1: - objs = objs[0] - - rv = 0 - try: - it = iter(objs) - rv = next(it) - for value in it: - rv = (rv << shift) ^ value - except TypeError: - return objs - except StopIteration: - pass - return rv - -def hash_objs(*objs, **kwargs): - return hash(tuple(objs)) diff --git a/tensilelite/Tensile/__init__.py b/tensilelite/Tensile/__init__.py index 517ff74ced..7f712e62be 100644 --- a/tensilelite/Tensile/__init__.py +++ b/tensilelite/Tensile/__init__.py @@ -24,13 +24,16 @@ # Even though we don't support python 2, this is still packaged sometimes with python 2. from __future__ import print_function +from os import path -# hardcoded tensile version; also in Tensile/Source/TensileConfigVersion.cmake +# Hardcoded tensilelite version +# Also in Tensile/Source/TensileConfigVersion.cmake __version__ = "4.33.0" -def PrintTensileRoot(): - import os.path - - print(os.path.dirname(__file__), end='') +ROOT_PATH: str = path.dirname(__file__) +SOURCE_PATH: str = path.join(ROOT_PATH, "Source") +def PrintTensileRoot(): + print(ROOT_PATH, end='') +__all__ = ["__version__", "ROOT_PATH", "SOURCE_PATH"] \ No newline at end of file diff --git a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake index 2a24d18c48..a70d403c64 100644 --- a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake +++ b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake @@ -22,7 +22,7 @@ # ################################################################################ -# hardcoded tensile version; also in Tensile/__init__.py +# hardcoded tensile version; also in Tensile/Common/Common.py set(TENSILE_VERSION_MAJOR 4) set(TENSILE_VERSION_MINOR 33) set(TENSILE_VERSION_PATCH 0) diff --git a/tensilelite/bump-version.sh b/tensilelite/bump-version.sh index a8268fa581..acebdbeb5c 100755 --- a/tensilelite/bump-version.sh +++ b/tensilelite/bump-version.sh @@ -33,7 +33,7 @@ NEW_VERSION="4.33.0" OLD_MINIMUM_REQUIRED_VERSION="MinimumRequiredVersion: 4.7.2" NEW_MINIMUM_REQUIRED_VERSION="MinimumRequiredVersion: 4.8.0" -sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" Tensile/__init__.py +sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" Tensile/Common/Common.py sed -i "s/${OLD_VERSION}/${NEW_VERSION}/g" HostLibraryTests/CMakeLists.txt echo "The version number also needs to be fixed in Tensile/cmake/TensileConfigVersion.cmake ." diff --git a/tensilelite/tox.ini b/tensilelite/tox.ini index 3eb9b377a0..81d82e6384 100644 --- a/tensilelite/tox.ini +++ b/tensilelite/tox.ini @@ -7,7 +7,6 @@ envlist = py35,py36,py27,lint # process to crash if a multiprocessing job has an exception. Fixed in 5.3.3. deps = -r{toxinidir}/requirements.txt - joblib pytest>=5.4.1 pytest-xdist>=1.32.0 filelock From e745693cf10fc7a3a9d9e0541a87af6124f9fd79 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 28 Jan 2025 14:32:13 -0600 Subject: [PATCH 020/105] feat: remove 'CustomKernelDirectory' global param --- tensilelite/Tensile/BenchmarkProblems.py | 6 ++---- tensilelite/Tensile/Common/Common.py | 5 +---- tensilelite/Tensile/CustomKernels.py | 15 ++++++++------- tensilelite/Tensile/KernelWriterAssembly.py | 3 ++- tensilelite/Tensile/__init__.py | 1 + 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 20708b51bc..ce4d30c3a4 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -30,9 +30,7 @@ from copy import deepcopy from pathlib import Path -from . import ClientExecutable -from . import SolutionLibrary -from . import LibraryIO +from . import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni @@ -69,7 +67,7 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo return solutions -def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, directory=globalParameters["CustomKernelDirectory"]): +def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, directory=CUSTOM_KERNEL_PATH): """Creates the Solution object for a custom kernel""" config = getCustomKernelConfig(kernelName, internalSupportParams, directory) return Solution(config, cxxCompiler) diff --git a/tensilelite/Tensile/Common/Common.py b/tensilelite/Tensile/Common/Common.py index a224e1615d..953073a52b 100644 --- a/tensilelite/Tensile/Common/Common.py +++ b/tensilelite/Tensile/Common/Common.py @@ -259,9 +259,6 @@ # control if a solution is run for a given problem globalParameters["GranularityThreshold"] = 0.0 -# directory where custom kernels are located -globalParameters["CustomKernelDirectory"] = os.path.join(os.path.dirname(os.path.realpath(__file__)), "CustomKernels") - globalParameters["PristineOnGPU"] = True # use Pristine memory on Tensile trainning verification or not globalParameters["SeparateArchitectures"] = False # write Tensile library metadata to separate files for each architecture @@ -1073,7 +1070,7 @@ def getArchitectureName(gfxName): "KernelLanguage": [ "Assembly" ], "ISA": validISA, # arch for assembly kernels - # Name of the custom kernel located in globalParameters["CustomKernelDirectory"]. + # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. # Ex: diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index e7a9dbce9c..12a8214eb9 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -22,7 +22,8 @@ # ################################################################################ -from .Common import globalParameters, checkParametersAreValid, validParameters +from . import CUSTOM_KERNEL_PATH +from .Common import checkParametersAreValid, validParameters import yaml @@ -31,20 +32,20 @@ def isCustomKernelConfig(config): return "CustomKernelName" in config and config["CustomKernelName"] -def getCustomKernelFilepath(name, directory=globalParameters["CustomKernelDirectory"]): +def getCustomKernelFilepath(name, directory=CUSTOM_KERNEL_PATH): return os.path.join(directory, (name + ".s")) -def getAllCustomKernelNames(directory=globalParameters["CustomKernelDirectory"]): +def getAllCustomKernelNames(directory=CUSTOM_KERNEL_PATH): return [fname[:-2] for fname in os.listdir(directory) if fname.endswith(".s")] -def getCustomKernelContents(name, directory=globalParameters["CustomKernelDirectory"]): +def getCustomKernelContents(name, directory=CUSTOM_KERNEL_PATH): try: with open(getCustomKernelFilepath(name, directory)) as f: return f.read() except: raise RuntimeError("Failed to find custom kernel: {}".format(os.path.join(directory, name))) -def getCustomKernelConfigAndAssembly(name, directory=globalParameters["CustomKernelDirectory"]): +def getCustomKernelConfigAndAssembly(name, directory=CUSTOM_KERNEL_PATH): contents = getCustomKernelContents(name, directory) config = "\n" #Yaml configuration properties assembly = "" @@ -57,14 +58,14 @@ def getCustomKernelConfigAndAssembly(name, directory=globalParameters["CustomKer return (config, assembly) -def readCustomKernelConfig(name, directory=globalParameters["CustomKernelDirectory"]): +def readCustomKernelConfig(name, directory=CUSTOM_KERNEL_PATH): rawConfig, _ = getCustomKernelConfigAndAssembly(name, directory) try: return yaml.safe_load(rawConfig)["custom.config"] except yaml.scanner.ScannerError as e: raise RuntimeError("Failed to read configuration for custom kernel: {0}\nDetails:\n{1}".format(name, e)) -def getCustomKernelConfig(kernelName, internalSupportParams, directory=globalParameters["CustomKernelDirectory"]): +def getCustomKernelConfig(kernelName, internalSupportParams, directory=CUSTOM_KERNEL_PATH): kernelConfig = readCustomKernelConfig(kernelName, directory) if "InternalSupportParams" not in kernelConfig: raise RuntimeError("Custom kernel %s config must have KernArgsVersion"%kernelName) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 15790ecdd9..36d4738369 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -22,6 +22,7 @@ # ################################################################################ +from . import CUSTOM_KERNEL_PATH from .TensileInstructions import KernelBody, Label, Macro, Module, RegSet, SrdUpperValue, \ StructuredModule, TextBlock, ValueEndif, ValueIf, ValueSet, SignatureBase, \ MUBUFModifiers, RegisterContainer, InstType, SelectBit, SGetPositivePCOffset, \ @@ -77,7 +78,7 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: return (0, "") # should this be an non zero number try: - code = self._getCustomKernelSource(kernel, globalParameters["CustomKernelDirectory"]) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) + code = self._getCustomKernelSource(kernel, CUSTOM_KERNEL_PATH) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) errcode = 0 except RuntimeError as e: printWarning(f"Failed to generate assembly source code for {kernel}: {e}") diff --git a/tensilelite/Tensile/__init__.py b/tensilelite/Tensile/__init__.py index 7f712e62be..102dfe3d05 100644 --- a/tensilelite/Tensile/__init__.py +++ b/tensilelite/Tensile/__init__.py @@ -32,6 +32,7 @@ ROOT_PATH: str = path.dirname(__file__) SOURCE_PATH: str = path.join(ROOT_PATH, "Source") +CUSTOM_KERNEL_PATH: str = path.join(ROOT_PATH, "CustomKernels") def PrintTensileRoot(): print(ROOT_PATH, end='') From 20b514be8e707743136b5d9fedcfd1352717af5b Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 29 Jan 2025 13:27:09 -0600 Subject: [PATCH 021/105] fix: amd clang version global param --- tensilelite/Tensile/BenchmarkProblems.py | 8 ++++---- tensilelite/Tensile/Common/Types.py | 9 +++++++-- .../Tensile/Components/GlobalWriteBatch.py | 14 +++++++++----- tensilelite/Tensile/KernelWriter.py | 5 +++-- tensilelite/Tensile/KernelWriterAssembly.py | 9 +++++---- .../Tensile/TensileCreateLibrary/Run.py | 18 +++++++++++------- tensilelite/Tensile/Toolchain/Assembly.py | 4 +++- 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index ce4d30c3a4..837f3dbb89 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -41,7 +41,7 @@ from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain from .Common import globalParameters, HR, print1, print2, \ - printExit, printWarning, ensurePath, startTime, tqdm, state + printExit, printWarning, ensurePath, startTime, tqdm, state, SemanticVersion def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): @@ -111,7 +111,7 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, buildTmpPath: Path): + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -140,7 +140,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, srcToolchain.compiler) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] @@ -153,7 +153,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") - newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, srcToolchain.compiler) + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler) newLibrary.applyNaming(kernelMinNaming) LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index 5c887371f0..0f0daccc86 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -1,3 +1,8 @@ -from typing import Tuple +from typing import Tuple, NamedTuple -IsaVersion = Tuple[int, int, int] \ No newline at end of file +IsaVersion = Tuple[int, int, int] + +class SemanticVersion(NamedTuple): + major: int + minor: int + patch: int \ No newline at end of file diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index 6bd50cb677..fd00814fa3 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..Common import globalParameters, DataDirection +from ..Common import globalParameters, DataDirection, SemanticVersion from ..Component import GlobalWriteComponents from ..SolutionStructs import Solution from ..Activation import ActivationModule, ActivationType @@ -41,19 +41,19 @@ def __call__(self, kernel: Solution, tPA, tPB, activation: ActivationModule, ss: batchIdx, applyAlpha, beta, edge, atomic, gwvw, atomicW, \ batchElements, addrE, addrD, addrC, addrBias, addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, isLocalBarrierInit: bool, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, batchElementSgprs, tmpSgpr, codeAccVgprRead, \ - codeMulAlpha, packdata, parentWriter, factorDim) -> Module: + codeMulAlpha, packdata, parentWriter, factorDim, amdClangVersion: SemanticVersion) -> Module: return GlobalWriteBatchWriter(kernel, tPA, tPB, activation, ss, batchIdx, applyAlpha, \ beta, edge, atomic, gwvw, atomicW, \ batchElements, addrE, addrD, addrC, addrBias, addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, isLocalBarrierInit, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, batchElementSgprs, tmpSgpr, \ - codeAccVgprRead, codeMulAlpha, packdata, parentWriter, factorDim).emit() + codeAccVgprRead, codeMulAlpha, packdata, parentWriter, factorDim, amdClangVersion).emit() class GlobalWriteBatchWriter: def __init__(self, kernel: Solution, tPA, tPB, activation: ActivationModule, ss: StoreState, \ batchIdx, applyAlpha, beta, edge, atomic, gwvw, atomicW, \ batchElements, addrE, addrD, addrC, addrBias, addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, isLocalBarrierInit: bool, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, batchElementSgprs, tmpSgpr, codeAccVgprRead, \ - codeMulAlpha, packdata, parentWriter, factorDim): + codeMulAlpha, packdata, parentWriter, factorDim, amdClangVersion: SemanticVersion): self.kernel = kernel self.tPA = tPA self.tPB = tPB @@ -91,6 +91,7 @@ def __init__(self, kernel: Solution, tPA, tPB, activation: ActivationModule, ss: self.parentWriter = parentWriter self.storesIssued = 0 self.factorDim = factorDim + self.amdClangVersion = amdClangVersion # Internal state for GlobalWriteBatch # 0 for None, 1 for WorkGroupReduction = False, 2 for WorkGroupReduction = True @@ -2029,9 +2030,12 @@ def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, cvtV if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): if self.parentWriter.states.archCaps["VOP3ByteSel"]: sb = 0 if self.gwvw == 1 else 1 + # delete me clangver = globalParameters['AMDClangVersion'].split(".") clangMaj = int(clangver[0]) - if not (clangMaj >= 19): + assert clangMaj == self.cxxCompilerVersion.major + # delete me + if not self.amdClangVersion.major >= 19: module.add(VCvtFP8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(op_sel=[0,sb]))) else: module.add(VCvtFP8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(byte_sel=sb))) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 70e13a8370..0c86221fec 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,7 +36,7 @@ from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule -from .Common import globalParameters, printWarning, roundUp, print2, printExit, INDEX_CHARS, DataDirection +from .Common import globalParameters, printWarning, roundUp, print2, printExit, INDEX_CHARS, DataDirection, SemanticVersion import abc import os @@ -356,10 +356,11 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str): + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler + self.amdClangVersion = amdClangVersion self.ti = None self.do = {} diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 36d4738369..1e9338bf57 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,7 +48,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection +from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection, SemanticVersion from math import ceil, log, floor from copy import deepcopy @@ -67,8 +67,8 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler) + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion) def getSourceFileString(self, kernel) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" @@ -10647,7 +10647,8 @@ def globalWriteBatch(self, kernel, tPA, tPB, activation, ss: StoreState, batchId batchElements, addrE, addrD, addrC, addrBias, \ addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, biasLocalBarrierInit, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, \ - batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, packdata, self, factorDim) + batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, packdata, self, factorDim, \ + self.amdClangVersion) ############################################################################## def openPrefetchGlobalRead2(self, kernel): diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 18846e5d5c..86d224fa9b 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -38,7 +38,8 @@ from Tensile.Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults from Tensile.TensileInstructions import getGfxName, TensileInstructions from Tensile.Common import globalParameters, HR, print1, print2, printExit, IsaVersion, ensurePath, state, \ - CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2, tqdm + CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2, tqdm, \ + SemanticVersion from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H from Tensile.SolutionLibrary import MasterSolutionLibrary @@ -240,12 +241,11 @@ def assemble(ret): @timing -def getSolutionAndKernelWriters(solutions, kernels, assembler): - +def getSolutionAndKernelWriters(solutions, kernels, assembler: str, assemblerVersion: SemanticVersion): kernelSerialNaming = Solution.getSerialNaming(kernels) solutionMinNaming = Solution.getMinNaming(solutions) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, assembler) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion) return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) @@ -366,10 +366,14 @@ def run(): arguments = parseArguments() outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) - cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( - arguments["CxxCompiler"], arguments["CCompiler"], arguments["OffloadBundler"], arguments["Assembler"], ToolchainDefaults.HIP_CONFIG + arguments["CxxCompiler"], + arguments["CCompiler"], + arguments["OffloadBundler"], + arguments["Assembler"], + ToolchainDefaults.HIP_CONFIG, ) + hipVersion = getVersion(hipconfig, regex=r'(.+)') print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") @@ -428,7 +432,7 @@ def validLogicFile(p: Path): solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) - kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters(solutions, kernels, assembler) + kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters(solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion) copyStaticFiles(outputPath) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index c0de0a7fd9..6c568a95c9 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -33,13 +33,15 @@ from typing import List, Literal, Union from ..TensileInstructions import getGfxName -from ..Common import globalParameters, print2, ensurePath +from ..Common import globalParameters, print2, ensurePath, SemanticVersion from ..KernelWriterAssembly import KernelWriterAssembly +from ..Toolchain.Validators import getVersion from ..SolutionStructs import Solution class AssemblyToolchain: def __init__(self, assembler: str, bundler: str, buildIdKind: str, coVersion: Literal[4, 5]): self.assembler = assembler + self.assemblerVersion = SemanticVersion(*[int(c) for c in getVersion(assembler).split(".")[:3]]) self.bundler = bundler self.buildIdKind = buildIdKind self.coVersion = coVersion From 884777bbcea190cde4447c8171219f025e086234 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 30 Jan 2025 18:31:53 -0600 Subject: [PATCH 022/105] fix: second pass on amd clang version --- tensilelite/Tensile/Common/Common.py | 4 ++++ tensilelite/Tensile/Components/GlobalWriteBatch.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/Common/Common.py b/tensilelite/Tensile/Common/Common.py index 63703c10d6..24711189f8 100644 --- a/tensilelite/Tensile/Common/Common.py +++ b/tensilelite/Tensile/Common/Common.py @@ -231,7 +231,9 @@ globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi globalParameters["HipClangVersion"] = "0.0.0" +# delete me globalParameters["AMDClangVersion"] = "0.0.0" +# delete me # default runtime is selected based on operating system, user can override if os.name == "nt": @@ -1767,9 +1769,11 @@ def assignGlobalParameters(config, cxxCompiler=None): if 'HIP version' in line: globalParameters['HipClangVersion'] = line.split()[2] print1("# Found hipcc version " + globalParameters['HipClangVersion']) + # delete me if 'AMD clang version' in line: globalParameters['AMDClangVersion'] = line.split()[3] print1("# Found clang version " + globalParameters['AMDClangVersion']) + # delete me except (subprocess.CalledProcessError, OSError) as e: printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index f5fa219580..bb260fd45a 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -2034,7 +2034,7 @@ def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, cvtV # delete me clangver = globalParameters['AMDClangVersion'].split(".") clangMaj = int(clangver[0]) - assert clangMaj == self.cxxCompilerVersion.major + assert clangMaj == self.amdClangVersion.major # delete me if not self.amdClangVersion.major >= 19: module.add(VCvtFP8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(op_sel=[0,sb]))) @@ -2070,9 +2070,12 @@ def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, cvtV if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): if self.parentWriter.states.archCaps["VOP3ByteSel"]: sb = 0 if self.gwvw == 1 else 1 + # delete me clangver = globalParameters['AMDClangVersion'].split(".") clangMaj = int(clangver[0]) - if not (clangMaj >= 19): + assert clangMaj == self.amdClangVersion.major + # delete me + if not self.amdClangVersion.major >= 19: module.add(VCvtBF8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(op_sel=[0,sb]))) else: module.add(VCvtBF8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(byte_sel=sb))) From 044fdf62eed71219ac4d5baee80cdf79cf7fe565 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:10:23 -0600 Subject: [PATCH 023/105] fix: imports for bolted on ops scripts --- tensilelite/Tensile/Ops/AMaxGenerator.py | 2 +- tensilelite/Tensile/Ops/LayerNormGenerator.py | 2 +- tensilelite/Tensile/Ops/SoftmaxGenerator.py | 2 +- tensilelite/Tensile/tcl.log | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 tensilelite/Tensile/tcl.log diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index 8c94b681e4..9a217453bd 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -32,7 +32,7 @@ import collections from contextlib import contextmanager import Tensile.TensileInstructions as ti -from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index e1180cce6f..093a437544 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -32,7 +32,7 @@ import collections from contextlib import contextmanager import Tensile.TensileInstructions as ti -from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index 8b89663324..1078615d84 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -31,7 +31,7 @@ import subprocess from contextlib import contextmanager import Tensile.TensileInstructions as ti -from .Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ +from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ assignGlobalParameters, getGfxName, gfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain diff --git a/tensilelite/Tensile/tcl.log b/tensilelite/Tensile/tcl.log new file mode 100644 index 0000000000..1141235085 --- /dev/null +++ b/tensilelite/Tensile/tcl.log @@ -0,0 +1 @@ +zsh: no such file or directory: Tensile/bin/TensileCreateLibrary From 5ffa0a4ead330c02a0797b45e435b45c72cce2d5 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 4 Feb 2025 12:15:42 -0600 Subject: [PATCH 024/105] chore: remove safety asserts --- tensilelite/Tensile/Common/Common.py | 8 -------- tensilelite/Tensile/Components/GlobalWriteBatch.py | 10 ---------- tensilelite/Tensile/TensileCreateLibrary/Run.py | 2 -- 3 files changed, 20 deletions(-) diff --git a/tensilelite/Tensile/Common/Common.py b/tensilelite/Tensile/Common/Common.py index 24711189f8..2096b67814 100644 --- a/tensilelite/Tensile/Common/Common.py +++ b/tensilelite/Tensile/Common/Common.py @@ -231,9 +231,6 @@ globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi globalParameters["HipClangVersion"] = "0.0.0" -# delete me -globalParameters["AMDClangVersion"] = "0.0.0" -# delete me # default runtime is selected based on operating system, user can override if os.name == "nt": @@ -1769,11 +1766,6 @@ def assignGlobalParameters(config, cxxCompiler=None): if 'HIP version' in line: globalParameters['HipClangVersion'] = line.split()[2] print1("# Found hipcc version " + globalParameters['HipClangVersion']) - # delete me - if 'AMD clang version' in line: - globalParameters['AMDClangVersion'] = line.split()[3] - print1("# Found clang version " + globalParameters['AMDClangVersion']) - # delete me except (subprocess.CalledProcessError, OSError) as e: printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index bb260fd45a..e1c0b499c4 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -2031,11 +2031,6 @@ def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, cvtV if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): if self.parentWriter.states.archCaps["VOP3ByteSel"]: sb = 0 if self.gwvw == 1 else 1 - # delete me - clangver = globalParameters['AMDClangVersion'].split(".") - clangMaj = int(clangver[0]) - assert clangMaj == self.amdClangVersion.major - # delete me if not self.amdClangVersion.major >= 19: module.add(VCvtFP8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(op_sel=[0,sb]))) else: @@ -2070,11 +2065,6 @@ def _addSumAlphaWithCBeta(self, kernel, ss, gwvw, elementIdx, vc0, tmpVgpr, cvtV if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): if self.parentWriter.states.archCaps["VOP3ByteSel"]: sb = 0 if self.gwvw == 1 else 1 - # delete me - clangver = globalParameters['AMDClangVersion'].split(".") - clangMaj = int(clangver[0]) - assert clangMaj == self.amdClangVersion.major - # delete me if not self.amdClangVersion.major >= 19: module.add(VCvtBF8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), vop3=VOP3PModifiers(op_sel=[0,sb]))) else: diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index d938b9c6e0..18b6b879b1 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -396,8 +396,6 @@ def run(): arguments["Assembler"], ToolchainDefaults.HIP_CONFIG, ) - hipVersion = getVersion(hipconfig, regex=r'(.+)') - print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") print1(f"# C Compiler: {cCompiler} (version {getVersion(cCompiler)})") From 94e497352159725ba86b0b05c7db6d69a4bd4317 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 5 Feb 2025 13:28:58 -0600 Subject: [PATCH 025/105] chore: create more separation in Common --- tensilelite/Tensile/ClientExecutable.py | 14 +- tensilelite/Tensile/ClientWriter.py | 3 +- tensilelite/Tensile/Common/Architectures.py | 103 +++ tensilelite/Tensile/Common/Capabilities.py | 192 +++++ .../Common/{Common.py => GlobalParameters.py} | 669 +----------------- tensilelite/Tensile/Common/Parallel.py | 20 +- tensilelite/Tensile/Common/Utilities.py | 586 +++++++++++++++ tensilelite/Tensile/Common/__init__.py | 7 +- tensilelite/Tensile/Contractions.py | 4 +- tensilelite/Tensile/EmbeddedData.py | 4 +- tensilelite/Tensile/GenerateSummations.py | 21 +- tensilelite/Tensile/Hardware.py | 2 +- .../Tensile/KernelWriterActivationFunction.py | 4 +- tensilelite/Tensile/KernelWriterConversion.py | 4 +- tensilelite/Tensile/LibraryIO.py | 4 +- tensilelite/Tensile/Ops/AMaxGenerator.py | 4 +- tensilelite/Tensile/Ops/LayerNormGenerator.py | 4 +- tensilelite/Tensile/Ops/SoftmaxGenerator.py | 4 +- tensilelite/Tensile/SolutionLibrary.py | 2 +- tensilelite/Tensile/TensileClientConfig.py | 4 +- .../Tensile/TensileCreateLibrary/Run.py | 4 +- .../Tensile/TensileInstructions/Base.py | 211 +----- .../Tensile/TensileInstructions/Code.py | 3 +- .../Tensile/TensileInstructions/Utils.py | 6 +- tensilelite/Tensile/TensileRetuneLibrary.py | 8 +- tensilelite/Tensile/Toolchain/Assembly.py | 3 +- tensilelite/Tensile/bin/TensileValidateLogic | 44 ++ 27 files changed, 997 insertions(+), 937 deletions(-) create mode 100644 tensilelite/Tensile/Common/Architectures.py create mode 100644 tensilelite/Tensile/Common/Capabilities.py rename tensilelite/Tensile/Common/{Common.py => GlobalParameters.py} (77%) create mode 100644 tensilelite/Tensile/Common/Utilities.py create mode 100755 tensilelite/Tensile/bin/TensileValidateLogic diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index 563b5bde40..ec5934dc63 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -29,7 +29,7 @@ from pathlib import Path from . import SOURCE_PATH -from .Common import Common, globalParameters +from .Common import globalParameters, print2, ClientExecutionLock, ensurePath class CMakeEnvironment: def __init__(self, sourceDir, buildDir, **options): @@ -43,14 +43,14 @@ def generate(self): args += itertools.chain.from_iterable([ ['-D', '{}={}'.format(key, value)] for key,value in self.options.items()]) args += [self.sourceDir] - Common.print2(' '.join(args)) - with Common.ClientExecutionLock(): - subprocess.check_call(args, cwd=Common.ensurePath(self.buildDir)) + print2(' '.join(args)) + with ClientExecutionLock(): + subprocess.check_call(args, cwd=ensurePath(self.buildDir)) def build(self): args = ['make', '-j'] - Common.print2(' '.join(args)) - with Common.ClientExecutionLock(): + print2(' '.join(args)) + with ClientExecutionLock(): subprocess.check_call(args, cwd=self.buildDir) def builtPath(self, path, *paths): @@ -59,7 +59,7 @@ def builtPath(self, path, *paths): def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cCompiler: str): sourcedir = SOURCE_PATH - builddir = Common.ensurePath(builddir) + builddir = ensurePath(builddir) options = {'CMAKE_BUILD_TYPE': globalParameters["CMakeBuildType"], 'TENSILE_USE_MSGPACK': 'ON', diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index d23482cffd..428e67196d 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -34,8 +34,7 @@ from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO -from .TensileInstructions import getGfxName -from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock +from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, getGfxName from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py new file mode 100644 index 0000000000..eedcad7838 --- /dev/null +++ b/tensilelite/Tensile/Common/Architectures.py @@ -0,0 +1,103 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +import re +from typing import Optional + +from .Types import IsaVersion + +# Translate GPU targets to filter filenames in Tensile_LOGIC directory +architectureMap = { + 'all':'_', + 'gfx000':'none', + 'gfx803':'r9nano', + 'gfx900':'vega10', + 'gfx906':'vega20', 'gfx906:xnack+':'vega20', 'gfx906:xnack-':'vega20', + 'gfx908':'arcturus','gfx908:xnack+':'arcturus', 'gfx908:xnack-':'arcturus', + 'gfx90a':'aldebaran', 'gfx90a:xnack+':'aldebaran', 'gfx90a:xnack-':'aldebaran', + 'gfx940':'aquavanjaram', 'gfx940:xnack+':'aquavanjaram', 'gfx940:xnack-':'aquavanjaram', + 'gfx941':'aquavanjaram', 'gfx941:xnack+':'aquavanjaram', 'gfx941:xnack-':'aquavanjaram', + 'gfx942':'aquavanjaram', 'gfx942:xnack+':'aquavanjaram', 'gfx942:xnack-':'aquavanjaram', + 'gfx1010':'navi10', + 'gfx1011':'navi12', + 'gfx1012':'navi14', + 'gfx1030':'navi21', + 'gfx1100':'navi31', + 'gfx1101':'navi32', + 'gfx1102':'navi33', + 'gfx1200':'gfx1200', + 'gfx1201':'gfx1201', +} + +def getGfxArch(name: str) -> Optional[IsaVersion]: + """Extracts the ISA version from a given GPU architecture name. + + Args: + name: The gfx name of the GPU architecture (e.g., 'gfx906'). + + Returns: + An object representing the major, minor, and step version of the ISA. + Returns None if the name does not match the expected pattern. + """ + match = re.search(r'gfx([0-9a-fA-F]{3,})', name) + if not match: + return None + ipart = match.group(1) + step = int(ipart[-1], 16) + + ipart = ipart[:-1] + minor = int(ipart[-1]) + + ipart = ipart[:-1] + major = int(ipart) + return tuple((major, minor, step)) + +def getGfxName(arch: IsaVersion) -> str: + """Converts an ISA version to a gfx architecture name. + + Args: + arch: An object representing the major, minor, and step version of the ISA. + + Returns: + The name of the GPU architecture (e.g., 'gfx906'). + """ + # Convert last digit to hex because reasons + name = str(arch[0]) + str(arch[1]) + ('%x' % arch[2]) + return 'gfx' + ''.join(map(str,name)) + +def getGfxCommonName(gfxName: str) -> Optional[str]: + """Retrieves the common name for a given gfx architecture name. + + Args: + gfxName: The name of the GPU architecture (e.g., gfx1100). + + Returns: + The common name of the GPU architecture (e.g., navi31) if found in ``architectureMap``. + Returns None if the name is not found. + """ + if gfxName in architectureMap: + return architectureMap[gfxName] + else: + for archKey in architectureMap: + if gfxName in archKey: + return architectureMap[archKey] + return None \ No newline at end of file diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py new file mode 100644 index 0000000000..f64e3cdb64 --- /dev/null +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -0,0 +1,192 @@ +import subprocess + +from functools import lru_cache +from typing import Tuple + +from .Architectures import getGfxName + +def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmString: str, \ + debug: bool=False, *options) -> bool: + """ + Try to assemble the asmString for the specified target processor + Success is defined as assembler returning no error code or stderr/stdout + """ + options = list(options) + + if isaVersion[0] >= 10: + options += ['-mwavefrontsize64'] + + args = [str(assemblerPath), '-x', 'assembler', + '-target', 'amdgcn-amdhsa', + '-mcpu='+ getGfxName(isaVersion), + *options, + '-'] + + result = subprocess.run(args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = result.stdout.decode() + + if debug: + print("isaVersion: ", isaVersion) + print("asm_cmd:", ' '.join(args)) + print("asmString: ", asmString) + print("output: ", output) + print("return code: ", result.returncode) + + if output != "" or result.returncode != 0: + return False + return True + +######################################## +# Get Caps +######################################## + +@lru_cache() +def initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: + """ Determine assembler capabilities by testing short instructions sequences """ + rv = {} + rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) + rv["HasExplicitCO"] = _tryAssembler(isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug) + rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) + + rv["HasDirectToLds"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug) + rv["HasAddLshl"] = _tryAssembler(isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug) + rv["HasLshlOr"] = _tryAssembler(isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug) + rv["HasSMulHi"] = _tryAssembler(isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug) + + rv["HasMFMA_explictB"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug) + rv["HasMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug) or rv["HasMFMA_explictB"] + rv["HasMFMA_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) or _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) + rv["HasMFMA_bf16_1k"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", isDebug) + rv["HasMFMA_f8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) + rv["HasMFMA_b8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) + + rv["HasMFMA_xf32"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", isDebug) + rv["HasSMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", isDebug) + rv["HasWMMA"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) + rv["HasWMMA_V1"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) + rv["HasWMMA_V2"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) + + rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) + + rv["v_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug) + rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) + + rv["v_pk_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug) + rv["v_pk_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug) + + rv["v_pk_add_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug) + rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug) + + rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug) + + rv["v_mad_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) + rv["v_fma_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) + + rv["v_dot2_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug) + rv["v_dot2c_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) + + rv["v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug) + rv["v_dot4c_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug) + rv["VOP3v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug) + + rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) + rv["v_fma_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug) + rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) + + rv["v_fma_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug) + + rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) + + rv["HasAtomicAdd"] = _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", isDebug) + rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", isDebug) + rv["HasMUBUFConst"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug) + rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) + + rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) + + rv["HasNTModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", isDebug) + + rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) + + if _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(63)", isDebug): + rv["MaxVmcnt"] = 63 + elif _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(15)", isDebug): + rv["MaxVmcnt"] = 15 + else: + rv["MaxVmcnt"] = 0 + + # TODO- Need to query the max cap, just like vmcnt as well? + rv["MaxLgkmcnt"] = 15 + + rv["SupportedSource"] = True + + return rv + +@lru_cache() +def initArchCaps(isaVersion) -> dict: + rv = {} + rv["HasEccHalf"] = (isaVersion in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["Waitcnt0Disabled"] = (isaVersion in [(9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["SeparateVscnt"] = isaVersion[0] in (10, 11) + rv["SeparateLGKMcnt"] = isaVersion[0] == (12) + rv["SeparateVMcnt"] = isaVersion[0] == (12) + rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) + rv["HasWave32"] = isaVersion[0] in (10, 11, 12) + rv["HasAccCD"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["ArchAccUnifiedRegs"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["CrosslaneWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["ForceStoreSC1"] = (isaVersion in [(9,4,0), (9,4,1)]) + rv["TransOpWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) + rv["DSLow16NotPreserve"] = isaVersion[0] == (12) + rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) + rv["NoSDWA"] = isaVersion[0] == (12) + rv["VOP3ByteSel"] = isaVersion[0] == (12) + rv["HasFP8_OCP"] = isaVersion[0] == (12) + return rv + +def initRegisterCaps(isaVersion, archCaps) -> dict: + rv = {} + rv["MaxVgpr"] = 256 + # max allowed is 112 out of 112 , 6 is used by hardware 4 SGPRs are wasted + rv["MaxSgpr"] = 102 + + rv["PhysicalMaxVgpr"] = 512 + rv["PhysicalMaxSgpr"] = 800 + + if isaVersion[0] == 10: + rv["PhysicalMaxVgprCU"] = 1024 * 32 + elif isaVersion[0] == 11: + if isaVersion[2] == 2: + rv["PhysicalMaxVgprCU"] = 1024 * 32 + else: + rv["PhysicalMaxVgprCU"] = 1536 * 32 + elif isaVersion[0] == 12: + rv["PhysicalMaxVgprCU"] = 1536 * 32 + elif isaVersion[0] == 9: + if archCaps["ArchAccUnifiedRegs"]: + rv["PhysicalMaxVgprCU"] = 2048 * 64 + else: + rv["PhysicalMaxVgprCU"] = 1024 * 64 + elif isaVersion[0] == 8: + rv["PhysicalMaxVgprCU"] = 1024 * 64 + elif isaVersion[0] == 0: + rv["PhysicalMaxVgprCU"] = 0 + else: + assert 0, "No valid VGPR value for this platform" + + return rv + +def initAsmBugs(asmCaps) -> dict: + rv = {} + rv["ExplicitCO"] = asmCaps["HasExplicitCO"] + rv["ExplicitNC"] = asmCaps["HasExplicitNC"] + + return rv \ No newline at end of file diff --git a/tensilelite/Tensile/Common/Common.py b/tensilelite/Tensile/Common/GlobalParameters.py similarity index 77% rename from tensilelite/Tensile/Common/Common.py rename to tensilelite/Tensile/Common/GlobalParameters.py index 2096b67814..e4331c76fe 100644 --- a/tensilelite/Tensile/Common/Common.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -22,19 +22,13 @@ # ################################################################################ -from enum import Enum -from collections import OrderedDict -from copy import deepcopy - import math import os.path -import subprocess -import sys -import functools import time -import re -from Tensile.TensileInstructions import getGfxName, TensileInstructions +from collections import OrderedDict +from copy import deepcopy + from Tensile import __version__ startTime = time.time() @@ -281,28 +275,8 @@ # Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen defaultGlobalParameters = deepcopy(globalParameters) -# Translate GPU targets to filter filenames in Tensile_LOGIC directory -architectureMap = { - 'all':'_','gfx000':'none', 'gfx803':'r9nano', 'gfx900':'vega10', - 'gfx906':'vega20', 'gfx906:xnack+':'vega20', 'gfx906:xnack-':'vega20', - 'gfx908':'arcturus','gfx908:xnack+':'arcturus', 'gfx908:xnack-':'arcturus', - 'gfx90a':'aldebaran', 'gfx90a:xnack+':'aldebaran', 'gfx90a:xnack-':'aldebaran', - 'gfx940':'aquavanjaram', 'gfx940:xnack+':'aquavanjaram', 'gfx940:xnack-':'aquavanjaram', - 'gfx941':'aquavanjaram', 'gfx941:xnack+':'aquavanjaram', 'gfx941:xnack-':'aquavanjaram', - 'gfx942':'aquavanjaram', 'gfx942:xnack+':'aquavanjaram', 'gfx942:xnack-':'aquavanjaram', - 'gfx1010':'navi10', 'gfx1011':'navi12', 'gfx1012':'navi14', 'gfx1030':'navi21', - 'gfx1100':'navi31', 'gfx1101':'navi32', 'gfx1102':'navi33', - 'gfx1200':'gfx1200', 'gfx1201':'gfx1201', -} -def getArchitectureName(gfxName): - if gfxName in architectureMap: - return architectureMap[gfxName] - else: - for archKey in architectureMap: - if gfxName in archKey: - return architectureMap[archKey] - return None + ################################################################################ # Tensile internal parameters @@ -1376,638 +1350,3 @@ def getArchitectureName(gfxName): "LibraryType": "GridBased", "SolutionImportanceMin": 0.01, # = 0.01=1% total time saved by keeping this solution } - - -################################################################################ -# Searching Nested Lists / Dictionaries -# to see if keys exist and what their values are -################################################################################ -# param name in structures? -def inListOfDictionaries(param, dictionaries): - for dictionary in dictionaries: - if param in dictionary: - return True - return False -def inListOfListOfDictionaries(param, dictionaries): - for dictionaryList in dictionaries: - if inListOfDictionaries(param, dictionaryList): - return True - return False -def inListOfLists(param, lists): - for l in lists: - if param in l: - return True - return False - -# get param values from structures. -def hasParam( name, structure ): - if isinstance(structure, list): - for l in structure: - if hasParam(name, l): - return True - return False - elif isinstance(structure, dict): - return name in structure - else: - return name == structure - #printExit("structure %s is not list or dict" % structure) - -def getParamValues( name, structure ): - if isinstance(structure, list): - for l in structure: - param = getParamValues(name, l) - if param != None: - return param - return None - elif isinstance(structure, dict): - if name in structure: - return structure[name] - else: - return None - else: - printExit("structure %s is not list or dict" % structure) - -################################################################################ -# Print Debug -################################################################################ -def print1(message): - if globalParameters["PrintLevel"] >= 1: - print(message) - sys.stdout.flush() -def print2(message): - if globalParameters["PrintLevel"] >= 2: - print(message) - sys.stdout.flush() - -def printWarning(message): - print("Tensile::WARNING: %s" % message) - sys.stdout.flush() -def printExit(message): - print("Tensile::FATAL: %s" % message) - sys.stdout.flush() - sys.exit(-1) - -################################################################################ -# Locate Executables -# rocm-smi, hip-clang, rocm_agent_enumerator, clang-offload-bundler -################################################################################ -def isExe( filePath ): - return os.path.isfile(filePath) and os.access(filePath, os.X_OK) -def locateExe( defaultPath, exeName ): # /opt/rocm/bin, hip-clang - # look in defaultPath first - exePath = os.path.join(defaultPath, exeName) - if isExe(exePath): - return exePath - # look in PATH second - for path in os.environ["PATH"].split(os.pathsep): - exePath = os.path.join(path, exeName) - if isExe(exePath): - return exePath - return None - -def gfxArch(name): - import re - match = re.search(r'gfx([0-9a-fA-F]{3,})', name) - if not match: return None - - ipart = match.group(1) - - step = int(ipart[-1], 16) - ipart = ipart[:-1] - - minor = int(ipart[-1]) - ipart = ipart[:-1] - - major = int(ipart) - - rv = (major, minor, step) - - return rv - - -def detectGlobalCurrentISA_(detectionTool): - """ - Returns returncode if detection failure - """ - global globalParameters - - if globalParameters["CurrentISA"] == (0,0,0) and detectionTool: - process = subprocess.run([detectionTool], stdout=subprocess.PIPE) - if os.name == "nt": - line = "" - for line_in in process.stdout.decode().splitlines(): - if 'gcnArchName' in line_in: - line += line_in.split()[1] - break # detemine if hipinfo will support multiple arch - arch = gfxArch(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + getGfxName(arch)) - globalParameters["CurrentISA"] = arch - else: - archList = [] - for line in process.stdout.decode().split("\n"): - arch = gfxArch(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + getGfxName(arch)) - archList.append(arch) - if len(archList) > 0: - globalParameters["CurrentISA"] = archList[globalParameters["Device"]] - if (process.returncode): - printWarning("%s exited with code %u" % (detectionTool, process.returncode)) - return process.returncode - return 0 - - -def detectGlobalCurrentISA(): - """ - Returns returncode if detection failure - """ - errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) - if errorCode: - printWarning("Attempting to detect ISA with rocm_agent_enumerator") - return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) - return errorCode - -def restoreDefaultGlobalParameters(): - """ - Restores `globalParameters` back to defaults. - """ - global globalParameters - global defaultGlobalParameters - # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would - # result in dangling references, specifically in Tensile.Tensile(). - globalParameters.clear() - for key, value in deepcopy(defaultGlobalParameters).items(): - globalParameters[key] = value - -def printTable(rows): - rows = list([[str(cell) for cell in row] for row in rows]) - colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - - for row in rows: - for (width, cell) in zip(colWidths, row): - pad = ' ' * (width - len(cell)) - print(pad, cell, sep='', end=' ') - print() - -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit("Invalid parameter name: {}\nValid parameters are {}." \ - .format(name, sorted(validParameters.keys()))) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ - if len(validParams[name])>32 else "" - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) - -def printCapTable(parameters): - import itertools - archs = [(0,0,0)] + parameters["SupportedISA"] - gfxNames = list(map(getGfxName, archs)) - - headerRow = ['cap'] + gfxNames - - def capRow(caps, cap): - return [cap] + [('1' if cap in caps[arch] and caps[arch][cap] else '0') for arch in archs] - - allAsmCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()])) - allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) - asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] - - allArchCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()])) - allArchCaps = sorted(allArchCaps) - archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] - - printTable([headerRow] + asmCapRows + archCapRows) - -def which(p): - if 'CMAKE_CXX_COMPILER' in os.environ and os.path.isfile(os.environ['CMAKE_CXX_COMPILER']): - return os.environ['CMAKE_CXX_COMPILER'] - if os.name == "nt": - exes = [p+x for x in ['.exe', '', '.bat']] # bat may be front end for file with no extension - else: - exes = [p+x for x in ['', '.exe', '.bat']] - system_path = os.environ['PATH'].split(os.pathsep) - for dirname in system_path+[globalParameters["ROCmBinPath"]]: - for exe in exes: - candidate = os.path.join(os.path.expanduser(dirname), exe) - if os.path.isfile(candidate): - return candidate - return None - -def splitArchs(fromTensile=False): - # Helper for architecture - def isSupported(arch): - return globalParameters["AsmCaps"][arch]["SupportedISA"] and \ - globalParameters["AsmCaps"][arch]["SupportedSource"] - - if ";" in globalParameters["Architecture"]: - wantedArchs = globalParameters["Architecture"].split(";") - else: - wantedArchs = globalParameters["Architecture"].split("_") - archs = [] - cmdlineArchs = [] - if "all" in wantedArchs: - for arch in globalParameters['SupportedISA']: - if isSupported(arch): - if (arch in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]): - if (arch == (9,0,10)): - archs += [getGfxName(arch) + '-xnack+'] - cmdlineArchs += [getGfxName(arch) + ':xnack+'] - if globalParameters["AsanBuild"]: - archs += [getGfxName(arch) + '-xnack+'] - cmdlineArchs += [getGfxName(arch) + ':xnack+'] - else: - archs += [getGfxName(arch) + '-xnack-'] - cmdlineArchs += [getGfxName(arch) + ':xnack-'] - else: - archs += [getGfxName(arch)] - cmdlineArchs += [getGfxName(arch)] - else: - for arch in wantedArchs: - archs += [re.sub(":", "-", arch)] - cmdlineArchs += [arch] - - # if calling from the context of Tensile we only want the arch associated with the current ISA - if fromTensile: - gfx = getGfxName(globalParameters["CurrentISA"]) - archs = set(a for a in archs if gfx in a) - cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) - - return archs, cmdlineArchs - -################################################################################ -################################################################################ -def assignGlobalParameters(config, cxxCompiler=None): - """ - Assign Global Parameters - Each global parameter has a default parameter, and the user - can override them, those overridings happen here - """ - - global globalParameters - - # Minimum Required Version - if "MinimumRequiredVersion" in config: - if not versionIsCompatible(config["MinimumRequiredVersion"]): - printExit("Config file requires version=%s is not compatible with current Tensile version=%s" \ - % (config["MinimumRequiredVersion"], __version__) ) - - # User-specified global parameters - print2("GlobalParameters:") - for key in globalParameters: - defaultValue = globalParameters[key] - if key in config: - configValue = config[key] - if configValue == defaultValue: - print2(" %24s: %8s (same)" % (key, configValue)) - else: - print2(" %24s: %8s (overriden)" % (key, configValue)) - else: - print2(" %24s: %8s (unspecified)" % (key, defaultValue)) - - globalParameters["ROCmPath"] = "/opt/rocm" - if "ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") - if "TENSILE_ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") - if os.name == "nt" and "HIP_DIR" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM - globalParameters["CmakeCxxCompiler"] = None - if "CMAKE_CXX_COMPILER" in os.environ: - globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") - if "CMAKE_C_COMPILER" in os.environ: - globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") - - globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") - - # ROCm AMD GPU Arch Path - # ROCm Agent Enumerator Path - if os.name == "nt": - globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") - globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") - else: - globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch") - globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator") - - globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") - globalParameters["ROCmLdPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld") - - globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel") - - if "AMDGPUArchPath" in config: - globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] - - if "AsanBuild" in config: - globalParameters["AsanBuild"] = config["AsanBuild"] - - if "KeepBuildTmp" in config: - globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] - - if "CodeObjectVersion" in config: - globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - - # read current gfx version - returncode = detectGlobalCurrentISA() - if globalParameters["CurrentISA"] == (0,0,0): - printWarning("Did not detect SupportedISA: %s; cannot benchmark assembly kernels." % globalParameters["SupportedISA"]) - if returncode: - if os.name == "nt": - globalParameters["CurrentISA"] = (9,0,6) - printWarning("Failed to detect ISA so forcing (gfx906) on windows") - - globalParameters["AsmCaps"] = {} - globalParameters["ArchCaps"] = {} - globalParameters["AsmBugs"] = {} - - for v in globalParameters["SupportedISA"] + [(0,0,0)]: - ti = TensileInstructions() - ti.init(v, cxxCompiler, (globalParameters["PrintLevel"] >= 2)) - globalParameters["AsmCaps"][v] = ti.getAsmCaps() - globalParameters["ArchCaps"][v] = ti.getArchCaps() - globalParameters["AsmBugs"][v] = ti.getAsmBugs() - - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) - - globalParameters["SupportedISA"] = list([i for i in globalParameters["SupportedISA"] if globalParameters["AsmCaps"][i]["SupportedISA"]]) - - validParameters["ISA"] = [(0,0,0), *globalParameters["SupportedISA"]] - - # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future - # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of - # '0.0.0' will persist - - # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. - # The alternative would be to install the `distro` package. - # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution - - # The following try except block computes the hipcc version - try: - if os.name == "nt": - compileArgs = ['perl'] + [which('hipcc')] + ['--version'] - output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() - else: - compiler = "hipcc" - output = subprocess.run([compiler, "--version"], check=True, stdout=subprocess.PIPE).stdout.decode() - - for line in output.split('\n'): - if 'HIP version' in line: - globalParameters['HipClangVersion'] = line.split()[2] - print1("# Found hipcc version " + globalParameters['HipClangVersion']) - - except (subprocess.CalledProcessError, OSError) as e: - printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) - - # The following keys may be present in the config, but are not (or no longer) global parameters. - ignoreKeys = [ - "UseCompression", - "CxxCompiler", - "CCompiler", - "OffloadBundler", - "Assembler", - "LogicPath", - "LogicFilter", - "OutputPath", - "Experimental", - "GenSolTable" - ] - for key in config: - if key in ignoreKeys: - continue - value = config[key] - if key not in globalParameters: - printWarning("Global parameter %s = %s unrecognised." % ( key, value )) - globalParameters[key] = value - - -def setupRestoreClocks(): - import atexit - def restoreClocks(): - if globalParameters["PinClocks"]: - rsmi = globalParameters["ROCmSMIPath"] - subprocess.call([rsmi, "-d", "0", "--resetclocks"]) - subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) - atexit.register(restoreClocks) -setupRestoreClocks() - -################################################################################ -# Assign Parameters -# populate dst with src[key] else give it the default/backup value -################################################################################ -def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ - defaultDictionary): - if key in sourceDictionary: - destinationDictionary[key] = deepcopy(sourceDictionary[key]) - else: - destinationDictionary[key] = deepcopy(defaultDictionary[key]) - - -def ensurePath(path): - try: - os.makedirs(path) - except FileExistsError: - pass - except OSError: - printExit("Failed to create directory \"%s\" " % (path) ) - return path - - -def roundUp(f): - return (int)(math.ceil(f)) - -################################################################################ -# Is query version compatible with current version -# a yaml file is compatible with tensile if -# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step -################################################################################ -def versionIsCompatible(queryVersionString): - (qMajor, qMinor, qStep) = queryVersionString.split(".") - (tMajor, tMinor, tStep) = __version__.split(".") - - # major version must match exactly - if qMajor != tMajor: - return False - - # minor.patch version must be >= - if int(qMinor) > int(tMinor): - return False - if qMinor == tMinor: - if int(qStep) > int(tStep): - return False - return True - -def ClientExecutionLock(): - if not globalParameters["ClientExecutionLockPath"]: - return open(os.devnull) - - import filelock - return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) - -# convert python list to C++ initializer style syntax -def listToInitializer(l): - return "{" + ','.join(map(str, l)) + "}" - -################################################################################ -# Progress Bar Printing -# prints "||||" up to width -################################################################################ -class ProgressBar: - def __init__(self, maxValue, width=80): - self.char = '|' - self.maxValue = maxValue - self.width = width - self.maxTicks = self.width - 7 - - - self.priorValue = 0 - self.fraction = 0 - self.numTicks = 0 - self.createTime = time.time() - - def increment(self, value=1): - self.update(self.priorValue+value) - - def update(self, value): - currentFraction = 1.0 * value / self.maxValue - currentNumTicks = int(currentFraction * self.maxTicks) - if currentNumTicks > self.numTicks: - self.numTicks = currentNumTicks - self.fraction = currentFraction - self.printStatus() - self.priorValue = value - - def printStatus(self): - sys.stdout.write("\r") - sys.stdout.write("[%-*s] %3d%%" \ - % (self.maxTicks, self.char*self.numTicks, self.fraction*100) ) - if self.numTicks == self.maxTicks: - stopTime = time.time() - sys.stdout.write(" (%-.1f secs elapsed)\n"%(stopTime-self.createTime)) - sys.stdout.flush() - - def finish(self): pass - - -class DataDirection(Enum): - NONE = 0, - READ = 1, - WRITE = 2 - -class SpinnyThing: - def __init__(self): - self.chars = ['|', '/', '-', '\\'] - self.index = 0; - - def increment(self, value=1): - sys.stdout.write('\b' + self.chars[self.index]) - sys.stdout.flush() - self.index = (self.index + 1) % len(self.chars) - - def finish(self): - sys.stdout.write('\b*\n') - sys.stdout.flush() - -def iterate_progress(obj, *args, **kwargs): - try: - progress = ProgressBar(len(obj)) - except TypeError: - progress = SpinnyThing() - for o in obj: - yield o - progress.increment() - progress.finish() - -try: - from tqdm import tqdm -except ImportError: - tqdm = iterate_progress - -def state(obj): - if hasattr(obj, 'state'): - return obj.state() - - if hasattr(obj.__class__, 'StateKeys'): - rv = {} - for key in obj.__class__.StateKeys: - attr = key - if isinstance(key, tuple): - (key, attr) = key - rv[key] = state(getattr(obj, attr)) - return rv - - if isinstance(obj, dict): - return {k: state(v) for k, v in obj.items()} - - if isinstance(obj, (str, int, float)): - return obj - - try: - return [state(i) for i in obj] - except TypeError: - pass - - return obj - -def state_key_ordering(cls): - def tup(obj): - return tuple([getattr(obj, k) for k in cls.StateKeys]) - - def lt(a, b): - return tup(a) < tup(b) - def eq(a, b): - return tup(a) == tup(b) - - cls.__lt__ = lt - cls.__eq__ = eq - - return functools.total_ordering(cls) - -def hash_combine(*objs, **kwargs): - shift = 1 - if 'shift' in kwargs: - shift = kwargs['shift'] - - if len(objs) == 1: - objs = objs[0] - - rv = 0 - try: - it = iter(objs) - rv = next(it) - for value in it: - rv = (rv << shift) ^ value - except TypeError: - return objs - except StopIteration: - pass - return rv - -def hash_objs(*objs, **kwargs): - return hash(tuple(objs)) - - -from copy import copy -class Backup: - """RAII class to restore backed up fields from object""" - fields = {} - object = None - def __init__(self, object, **fields): - self.object = object - for k, v in fields.items(): - self.fields[k] = copy(v) - def __del__(self): - for k, v in self.fields.items(): - setattr(self.object, k, v) - diff --git a/tensilelite/Tensile/Common/Parallel.py b/tensilelite/Tensile/Common/Parallel.py index 0c0abdf704..6852746c55 100644 --- a/tensilelite/Tensile/Common/Parallel.py +++ b/tensilelite/Tensile/Common/Parallel.py @@ -30,7 +30,7 @@ from joblib import Parallel, delayed -from .Common import tqdm +from .Utilities import tqdm def joblibParallelSupportsGenerator(): import joblib @@ -39,7 +39,7 @@ def joblibParallelSupportsGenerator(): return Version(joblibVer) >= Version("1.4.0") def CPUThreadCount(enable=True): - from .Common import globalParameters + from .GlobalParameters import globalParameters if not enable: return 1 else: @@ -79,9 +79,9 @@ def apply_print_exception(item, *args): sys.stderr.flush() def OverwriteGlobalParameters(newGlobalParameters): - from . import Common - Common.globalParameters.clear() - Common.globalParameters.update(newGlobalParameters) + from . import GlobalParameters + GlobalParameters.globalParameters.clear() + GlobalParameters.globalParameters.update(newGlobalParameters) def ProcessingPool(enable=True, maxTasksPerChild=None): import multiprocessing @@ -93,8 +93,8 @@ def ProcessingPool(enable=True, maxTasksPerChild=None): return multiprocessing.dummy.Pool(1) if multiprocessing.get_start_method() == "spawn": - from . import Common - return multiprocessing.Pool(threadCount, initializer=OverwriteGlobalParameters, maxtasksperchild=maxTasksPerChild, initargs=(Common.globalParameters,)) + from . import GlobalParameters + return multiprocessing.Pool(threadCount, initializer=OverwriteGlobalParameters, maxtasksperchild=maxTasksPerChild, initargs=(GlobalParameters.globalParameters,)) else: return multiprocessing.Pool(threadCount, maxtasksperchild=maxTasksPerChild) @@ -110,7 +110,7 @@ def ParallelMap(function, objects, message="", enable=True, method=None, maxTask - `lambda x: x.imap` - lazy evaluation - `lambda x: x.imap_unordered` - lazy evaluation, does not preserve order of return value. """ - from .Common import globalParameters + from .GlobalParameters import globalParameters threadCount = CPUThreadCount(enable) pool = ProcessingPool(enable, maxTasksPerChild) @@ -154,7 +154,7 @@ def ParallelMap(function, objects, message="", enable=True, method=None, maxTask return rv def ParallelMapReturnAsGenerator(function, objects, message="", enable=True, multiArg=True): - from .Common import globalParameters + from .GlobalParameters import globalParameters threadCount = CPUThreadCount(enable) print("{0}Launching {1} threads...".format(message, threadCount)) @@ -180,7 +180,7 @@ def ParallelMap2(function, objects, message="", enable=True, multiArg=True, retu if return_as in ('generator', 'generator_unordered') and not joblibParallelSupportsGenerator(): return ParallelMapReturnAsGenerator(function, objects, message, enable, multiArg) - from .Common import globalParameters + from .GlobalParameters import globalParameters threadCount = CPUThreadCount(enable) if threadCount <= 1 and globalParameters["ShowProgressBar"]: diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py new file mode 100644 index 0000000000..4ed27a4c9e --- /dev/null +++ b/tensilelite/Tensile/Common/Utilities.py @@ -0,0 +1,586 @@ +import os +import re +import sys +import math +import time +import functools +import warnings +import subprocess + +from copy import deepcopy +from enum import Enum +from typing import Optional + +from .Capabilities import initAsmBugs, initArchCaps, initAsmCaps +from .Architectures import getGfxArch, getGfxName +from .GlobalParameters import globalParameters, defaultGlobalParameters, validParameters, __version__ + +# get param values from structures. +def hasParam( name, structure ): + if isinstance(structure, list): + for l in structure: + if hasParam(name, l): + return True + return False + elif isinstance(structure, dict): + return name in structure + else: + return name == structure + #printExit("structure %s is not list or dict" % structure) + +################################################################################ +# Print Debug +################################################################################ +def print1(message): + if globalParameters["PrintLevel"] >= 1: + print(message) + sys.stdout.flush() +def print2(message): + if globalParameters["PrintLevel"] >= 2: + print(message) + sys.stdout.flush() + +def printWarning(message): + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() +def printExit(message): + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) + +################################################################################ +# Locate Executables +# rocm-smi, hip-clang, rocm_agent_enumerator, clang-offload-bundler +################################################################################ +def isExe( filePath ): + return os.path.isfile(filePath) and os.access(filePath, os.X_OK) +def locateExe( defaultPath, exeName ): # /opt/rocm/bin, hip-clang + # look in defaultPath first + exePath = os.path.join(defaultPath, exeName) + if isExe(exePath): + return exePath + # look in PATH second + for path in os.environ["PATH"].split(os.pathsep): + exePath = os.path.join(path, exeName) + if isExe(exePath): + return exePath + return None + + +def which(p): + if 'CMAKE_CXX_COMPILER' in os.environ and os.path.isfile(os.environ['CMAKE_CXX_COMPILER']): + return os.environ['CMAKE_CXX_COMPILER'] + if os.name == "nt": + exes = [p+x for x in ['.exe', '', '.bat']] # bat may be front end for file with no extension + else: + exes = [p+x for x in ['', '.exe', '.bat']] + system_path = os.environ['PATH'].split(os.pathsep) + for dirname in system_path+[globalParameters["ROCmBinPath"]]: + for exe in exes: + candidate = os.path.join(os.path.expanduser(dirname), exe) + if os.path.isfile(candidate): + return candidate + return None + +def splitArchs(fromTensile=False): + # Helper for architecture + def isSupported(arch): + return globalParameters["AsmCaps"][arch]["SupportedISA"] and \ + globalParameters["AsmCaps"][arch]["SupportedSource"] + + if ";" in globalParameters["Architecture"]: + wantedArchs = globalParameters["Architecture"].split(";") + else: + wantedArchs = globalParameters["Architecture"].split("_") + archs = [] + cmdlineArchs = [] + if "all" in wantedArchs: + for arch in globalParameters['SupportedISA']: + if isSupported(arch): + if (arch in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]): + if (arch == (9,0,10)): + archs += [getGfxName(arch) + '-xnack+'] + cmdlineArchs += [getGfxName(arch) + ':xnack+'] + if globalParameters["AsanBuild"]: + archs += [getGfxName(arch) + '-xnack+'] + cmdlineArchs += [getGfxName(arch) + ':xnack+'] + else: + archs += [getGfxName(arch) + '-xnack-'] + cmdlineArchs += [getGfxName(arch) + ':xnack-'] + else: + archs += [getGfxName(arch)] + cmdlineArchs += [getGfxName(arch)] + else: + for arch in wantedArchs: + archs += [re.sub(":", "-", arch)] + cmdlineArchs += [arch] + + # if calling from the context of Tensile we only want the arch associated with the current ISA + if fromTensile: + gfx = getGfxName(globalParameters["CurrentISA"]) + archs = set(a for a in archs if gfx in a) + cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) + + return archs, cmdlineArchs + + +def ensurePath(path): + try: + os.makedirs(path) + except FileExistsError: + pass + except OSError: + printExit("Failed to create directory \"%s\" " % (path) ) + return path + + +def roundUp(f): + return (int)(math.ceil(f)) + +################################################################################ +# Is query version compatible with current version +# a yaml file is compatible with tensile if +# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step +################################################################################ +def versionIsCompatible(queryVersionString): + (qMajor, qMinor, qStep) = queryVersionString.split(".") + (tMajor, tMinor, tStep) = __version__.split(".") + + # major version must match exactly + if qMajor != tMajor: + return False + + # minor.patch version must be >= + if int(qMinor) > int(tMinor): + return False + if qMinor == tMinor: + if int(qStep) > int(tStep): + return False + return True + + +def checkParametersAreValid(param, validParams): + """Ensures paramaters in params exist and have valid values as specified by validParames""" + (name, values) = param + if name == "ProblemSizes": + return + elif name == "InternalSupportParams": + return + + if name not in validParams: + printExit("Invalid parameter name: {}\nValid parameters are {}." \ + .format(name, sorted(validParameters.keys()))) + + for value in values: + if validParams[name] != -1 and value not in validParams[name]: + msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." + msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ + if len(validParams[name])>32 else "" + printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) + +################################################################################ +# Progress Bar Printing +# prints "||||" up to width +################################################################################ +class ProgressBar: + def __init__(self, maxValue, width=80): + self.char = '|' + self.maxValue = maxValue + self.width = width + self.maxTicks = self.width - 7 + + + self.priorValue = 0 + self.fraction = 0 + self.numTicks = 0 + self.createTime = time.time() + + def increment(self, value=1): + self.update(self.priorValue+value) + + def update(self, value): + currentFraction = 1.0 * value / self.maxValue + currentNumTicks = int(currentFraction * self.maxTicks) + if currentNumTicks > self.numTicks: + self.numTicks = currentNumTicks + self.fraction = currentFraction + self.printStatus() + self.priorValue = value + + def printStatus(self): + sys.stdout.write("\r") + sys.stdout.write("[%-*s] %3d%%" \ + % (self.maxTicks, self.char*self.numTicks, self.fraction*100) ) + if self.numTicks == self.maxTicks: + stopTime = time.time() + sys.stdout.write(" (%-.1f secs elapsed)\n"%(stopTime-self.createTime)) + sys.stdout.flush() + + def finish(self): pass + + +class DataDirection(Enum): + NONE = 0, + READ = 1, + WRITE = 2 + +class SpinnyThing: + def __init__(self): + self.chars = ['|', '/', '-', '\\'] + self.index = 0 + + def increment(self, value=1): + sys.stdout.write('\b' + self.chars[self.index]) + sys.stdout.flush() + self.index = (self.index + 1) % len(self.chars) + + def finish(self): + sys.stdout.write('\b*\n') + sys.stdout.flush() + +def iterate_progress(obj, *args, **kwargs): + try: + progress = ProgressBar(len(obj)) + except TypeError: + progress = SpinnyThing() + for o in obj: + yield o + progress.increment() + progress.finish() + +try: + from tqdm import tqdm +except ImportError: + tqdm = iterate_progress + +def state(obj): + if hasattr(obj, 'state'): + return obj.state() + + if hasattr(obj.__class__, 'StateKeys'): + rv = {} + for key in obj.__class__.StateKeys: + attr = key + if isinstance(key, tuple): + (key, attr) = key + rv[key] = state(getattr(obj, attr)) + return rv + + if isinstance(obj, dict): + return {k: state(v) for k, v in obj.items()} + + if isinstance(obj, (str, int, float)): + return obj + + try: + return [state(i) for i in obj] + except TypeError: + pass + + return obj + +def state_key_ordering(cls): + def tup(obj): + return tuple([getattr(obj, k) for k in cls.StateKeys]) + + def lt(a, b): + return tup(a) < tup(b) + def eq(a, b): + return tup(a) == tup(b) + + cls.__lt__ = lt + cls.__eq__ = eq + + return functools.total_ordering(cls) + +def hash_combine(*objs, **kwargs): + shift = 1 + if 'shift' in kwargs: + shift = kwargs['shift'] + + if len(objs) == 1: + objs = objs[0] + + rv = 0 + try: + it = iter(objs) + rv = next(it) + for value in it: + rv = (rv << shift) ^ value + except TypeError: + return objs + except StopIteration: + pass + return rv + +def hash_objs(*objs, **kwargs): + return hash(tuple(objs)) + +def showwarning(message, category, filename, lineno, file=None, line=None): + msg = f"> {category.__name__}: {message}" + print(msg) +warnings.showwarning = showwarning + + +################################################################################ +# Is query version compatible with current version +# a yaml file is compatible with tensile if +# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step +################################################################################ +def detectGlobalCurrentISA_(detectionTool): + """ + Returns returncode if detection failure + """ + global globalParameters + + if globalParameters["CurrentISA"] == (0,0,0) and detectionTool: + process = subprocess.run([detectionTool], stdout=subprocess.PIPE) + if os.name == "nt": + line = "" + for line_in in process.stdout.decode().splitlines(): + if 'gcnArchName' in line_in: + line += line_in.split()[1] + break # detemine if hipinfo will support multiple arch + arch = getGfxArch(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + getGfxName(arch)) + globalParameters["CurrentISA"] = arch + else: + archList = [] + for line in process.stdout.decode().split("\n"): + arch = getGfxArch(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + getGfxName(arch)) + archList.append(arch) + if len(archList) > 0: + globalParameters["CurrentISA"] = archList[globalParameters["Device"]] + if (process.returncode): + printWarning("%s exited with code %u" % (detectionTool, process.returncode)) + return process.returncode + return 0 + + +def detectGlobalCurrentISA(): + """ + Returns returncode if detection failure + """ + errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) + if errorCode: + printWarning("Attempting to detect ISA with rocm_agent_enumerator") + return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) + return errorCode + + +def restoreDefaultGlobalParameters(): + """ + Restores `globalParameters` back to defaults. + """ + global globalParameters + global defaultGlobalParameters + # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would + # result in dangling references, specifically in Tensile.Tensile(). + globalParameters.clear() + for key, value in deepcopy(defaultGlobalParameters).items(): + globalParameters[key] = value + +def printTable(rows): + rows = list([[str(cell) for cell in row] for row in rows]) + colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) + + for row in rows: + for (width, cell) in zip(colWidths, row): + pad = ' ' * (width - len(cell)) + print(pad, cell, sep='', end=' ') + print() + + +def printCapTable(parameters): + import itertools + archs = [(0,0,0)] + parameters["SupportedISA"] + gfxNames = list(map(getGfxName, archs)) + + headerRow = ['cap'] + gfxNames + + def capRow(caps, cap): + return [cap] + [('1' if cap in caps[arch] and caps[arch][cap] else '0') for arch in archs] + + allAsmCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()])) + allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) + asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + + allArchCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()])) + allArchCaps = sorted(allArchCaps) + archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + + printTable([headerRow] + asmCapRows + archCapRows) + + +def assignGlobalParameters(config, cxxCompiler=None): + """ + Assign Global Parameters + Each global parameter has a default parameter, and the user + can override them, those overridings happen here + """ + + global globalParameters + + # Minimum Required Version + if "MinimumRequiredVersion" in config: + if not versionIsCompatible(config["MinimumRequiredVersion"]): + printExit("Config file requires version=%s is not compatible with current Tensile version=%s" \ + % (config["MinimumRequiredVersion"], __version__) ) + + # User-specified global parameters + print2("GlobalParameters:") + for key in globalParameters: + defaultValue = globalParameters[key] + if key in config: + configValue = config[key] + if configValue == defaultValue: + print2(" %24s: %8s (same)" % (key, configValue)) + else: + print2(" %24s: %8s (overriden)" % (key, configValue)) + else: + print2(" %24s: %8s (unspecified)" % (key, defaultValue)) + + globalParameters["ROCmPath"] = "/opt/rocm" + if "ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") + if "TENSILE_ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") + if os.name == "nt" and "HIP_DIR" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM + globalParameters["CmakeCxxCompiler"] = None + if "CMAKE_CXX_COMPILER" in os.environ: + globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") + if "CMAKE_C_COMPILER" in os.environ: + globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") + + globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") + + # ROCm AMD GPU Arch Path + # ROCm Agent Enumerator Path + if os.name == "nt": + globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") + globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") + else: + globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch") + globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator") + + globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") + globalParameters["ROCmLdPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld") + + globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel") + + if "AMDGPUArchPath" in config: + globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] + + if "AsanBuild" in config: + globalParameters["AsanBuild"] = config["AsanBuild"] + + if "KeepBuildTmp" in config: + globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] + + if "CodeObjectVersion" in config: + globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] + + # read current gfx version + returncode = detectGlobalCurrentISA() + if globalParameters["CurrentISA"] == (0,0,0): + printWarning("Did not detect SupportedISA: %s; cannot benchmark assembly kernels." % globalParameters["SupportedISA"]) + if returncode: + if os.name == "nt": + globalParameters["CurrentISA"] = (9,0,6) + printWarning("Failed to detect ISA so forcing (gfx906) on windows") + + globalParameters["AsmCaps"] = {} + globalParameters["ArchCaps"] = {} + globalParameters["AsmBugs"] = {} + + for v in globalParameters["SupportedISA"] + [(0,0,0)]: + globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) + globalParameters["ArchCaps"][v] = initArchCaps(v) + globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + + if globalParameters["PrintLevel"] >= 1: + printCapTable(globalParameters) + + globalParameters["SupportedISA"] = list([i for i in globalParameters["SupportedISA"] if globalParameters["AsmCaps"][i]["SupportedISA"]]) + + validParameters["ISA"] = [(0,0,0), *globalParameters["SupportedISA"]] + + # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future + # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of + # '0.0.0' will persist + + # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. + # The alternative would be to install the `distro` package. + # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution + + # The following try except block computes the hipcc version + try: + if os.name == "nt": + compileArgs = ['perl'] + [which('hipcc')] + ['--version'] + output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() + else: + compiler = "hipcc" + output = subprocess.run([compiler, "--version"], check=True, stdout=subprocess.PIPE).stdout.decode() + + for line in output.split('\n'): + if 'HIP version' in line: + globalParameters['HipClangVersion'] = line.split()[2] + print1("# Found hipcc version " + globalParameters['HipClangVersion']) + + except (subprocess.CalledProcessError, OSError) as e: + printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) + + # The following keys may be present in the config, but are not (or no longer) global parameters. + ignoreKeys = [ + "UseCompression", + "CxxCompiler", + "CCompiler", + "OffloadBundler", + "Assembler", + "LogicPath", + "LogicFilter", + "OutputPath", + "Experimental", + "GenSolTable" + ] + for key in config: + if key in ignoreKeys: + continue + value = config[key] + if key not in globalParameters: + printWarning("Global parameter %s = %s unrecognised." % ( key, value )) + globalParameters[key] = value + +def setupRestoreClocks(): + import atexit + def restoreClocks(): + if globalParameters["PinClocks"]: + rsmi = globalParameters["ROCmSMIPath"] + subprocess.call([rsmi, "-d", "0", "--resetclocks"]) + subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) + atexit.register(restoreClocks) +setupRestoreClocks() + +################################################################################ +# Assign Parameters +# populate dst with src[key] else give it the default/backup value +################################################################################ +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ + defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) + +def ClientExecutionLock(): + if not globalParameters["ClientExecutionLockPath"]: + return open(os.devnull) + + import filelock + return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index 4d972028e4..07b36c3eed 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -1,7 +1,10 @@ -from .Common import * +from .Architectures import * +from .Utilities import * +from .GlobalParameters import * from .Constants import * +from .Capabilities import * from .Parallel import * from .Types import * # Dunder variables are not exported via `*` -from .Common import __version__ \ No newline at end of file +from .GlobalParameters import __version__ \ No newline at end of file diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 17a02e992e..6f61d65591 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -28,7 +28,7 @@ from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Common import Common, internalParameters, globalParameters, state, state_key_ordering +from .Common import getGfxArch, internalParameters, globalParameters, state, state_key_ordering @state_key_ordering class FreeIndex: @@ -699,7 +699,7 @@ def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): if 'ISA' not in d: if d['KernelLanguage'] == 'Assembly': - d['ISA'] = Common.gfxArch(deviceInfo[1]) + d['ISA'] = getGfxArch(deviceInfo[1]) else: d['ISA'] = [0,0,0] diff --git a/tensilelite/Tensile/EmbeddedData.py b/tensilelite/Tensile/EmbeddedData.py index b61cca966b..eb6bb21ce1 100644 --- a/tensilelite/Tensile/EmbeddedData.py +++ b/tensilelite/Tensile/EmbeddedData.py @@ -22,7 +22,7 @@ # ################################################################################ -from .Common import Common +from .Common import CHeader import itertools import os @@ -98,7 +98,7 @@ def end_namespace(self, name=None): ns.__exit__(None, None, None) def write_header(self): - self.write(Common.CHeader) + self.write(CHeader) self.write(self.includes) diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index a73ac3afe2..6e9bc57281 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -36,24 +36,12 @@ from . import LibraryIO from . import ClientWriter -from .TensileInstructions import getGfxName from .Common import assignGlobalParameters, ensurePath, globalParameters, \ - gfxArch, printExit, getArchitectureName + printExit, getGfxName, getGfxCommonName from .SolutionStructs import ProblemSizes -from .Utilities.Toolchain import ToolchainDefaults, validateToolchain +from .Toolchain.Validators import ToolchainDefaults, validateToolchain -def getArchitecture(isaName): - archid = getGfxName(isaName) - return getArchitectureName(archid) - -def isValidArch(archName, currentArch): - arch = gfxArch(archName) - return currentArch == arch - -############################################################################## -# createLibraryForBenchmark -############################################################################## def createLibraryForBenchmark(logicPath, libraryPath, currentPath): """ takes the path of existing logic files as input and adds the summation @@ -80,9 +68,10 @@ def GenerateSummations(userArgs): cxxCompiler, cCompiler = validateToolchain(ToolchainDefaults.CXX_COMPILER, ToolchainDefaults.C_COMPILER) currentISA = globalParameters["CurrentISA"] - currentArchitecture = getArchitecture(currentISA) + gfxName = getGfxName(currentISA) + commonName = getGfxCommonName(gfxName) - globPath = os.path.join(inputLogicPath, "{}*".format(currentArchitecture)) + globPath = os.path.join(inputLogicPath, "{}*".format(commonName)) logicFileNames = glob.glob(globPath) for logicFileName in logicFileNames: diff --git a/tensilelite/Tensile/Hardware.py b/tensilelite/Tensile/Hardware.py index 88ffc9a214..dd888a6281 100644 --- a/tensilelite/Tensile/Hardware.py +++ b/tensilelite/Tensile/Hardware.py @@ -23,7 +23,7 @@ ################################################################################ from . import Properties -from .TensileInstructions import getGfxName +from .Common import getGfxName import copy class HardwarePredicate(Properties.Predicate): diff --git a/tensilelite/Tensile/KernelWriterActivationFunction.py b/tensilelite/Tensile/KernelWriterActivationFunction.py index 95b8395e55..526178fe0c 100644 --- a/tensilelite/Tensile/KernelWriterActivationFunction.py +++ b/tensilelite/Tensile/KernelWriterActivationFunction.py @@ -23,7 +23,7 @@ from copy import deepcopy from .TensileInstructions import TensileInstructions -from .Common import globalParameters, CHeader, gfxArch, getGfxName +from .Common import globalParameters, getGfxArch, getGfxName from .Activation import ActivationInline, ActivationType from .KernelWriterBase import KernelWriterBase @@ -55,7 +55,7 @@ def __init__(self, state, cxxCompiler: str): self.supportedArchs = deepcopy(globalParameters['SupportedISA']) else: for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxArch(''.join(map(str, arch))) + self.supportedArchs[idx] = getGfxArch(''.join(map(str, arch))) # derive parameter self.language = "HIP" diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 8783337982..2755cd2274 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -27,7 +27,7 @@ from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, gfxArch, getGfxName, INDEX_CHARS +from .Common import globalParameters, getGfxArch, getGfxName, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): @@ -88,7 +88,7 @@ def __init__(self, state, load_vw): self.supportedArchs = deepcopy(globalParameters['SupportedISA']) else: for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxArch(''.join(map(str, arch))) + self.supportedArchs[idx] = getGfxArch(''.join(map(str, arch))) self.gsuKernels = [self.state["GlobalSplitU"]] if self.state["GenPGRPostKernels"]: diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index c5a9619d7d..7522b4c263 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -26,7 +26,7 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import Common, printExit, printWarning, print2, versionIsCompatible, __version__ +from .Common import getGfxArch, printExit, printWarning, print2, versionIsCompatible, __version__ from typing import NamedTuple, List import os @@ -273,7 +273,7 @@ def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): # unpack solution def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: if solutionState["KernelLanguage"] == "Assembly": - solutionState["ISA"] = Common.gfxArch(data["ArchitectureName"]) + solutionState["ISA"] = getGfxArch(data["ArchitectureName"]) else: solutionState["ISA"] = (0, 0, 0) solutionState["CUCount"] = data["CUCount"] diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index 9a217453bd..a56fb96205 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, gfxArch, globalParameters + assignGlobalParameters, getGfxName, getGfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -843,7 +843,7 @@ def meta_str(kernels: Tuple[KernelMeta]): debug_build: bool = args.debug_build arch: str = args.arch is_scale: bool = args.is_scale - isa = gfxArch(arch) + isa = getGfxArch(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index 093a437544..f57bbbbcdd 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, gfxArch, globalParameters + assignGlobalParameters, getGfxName, getGfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -920,7 +920,7 @@ def meta_str(kernels: Tuple[KernelMeta]): toolchain_path: str = validateToolchain(args.toolchain) debug_build: bool = args.debug_build arch: str = args.arch - isa = gfxArch(arch) + isa = getGfxArch(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index 1078615d84..d771aa7ac1 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -32,7 +32,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, gfxArch, globalParameters + assignGlobalParameters, getGfxName, getGfxArch, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def record_num_calls(f): @@ -687,7 +687,7 @@ def meta_str(kernels: Tuple[KernelMeta]): toolchain_path: str = validateToolchain(args.toolchain) debug_build: bool = args.debug_build arch: str = args.arch - isa = gfxArch(arch) + isa = getGfxArch(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index c8138b77cf..a9a6fb7850 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -314,7 +314,7 @@ def hardware(d, problemType, solutions, library, placeholderName): if devicePart == "fallback": pred = Hardware.HardwarePredicate("TruePred") else: - pred = Hardware.HardwarePredicate.FromHardware(Common.gfxArch(devicePart), cuCount) + pred = Hardware.HardwarePredicate.FromHardware(Common.getGfxArch(devicePart), cuCount) newLib.rows.append({"predicate": pred, "library": library}) diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py index 5c40efd179..da6d2f1377 100644 --- a/tensilelite/Tensile/TensileClientConfig.py +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -26,7 +26,7 @@ from . import LibraryIO from .Contractions import ProblemType as ContractionsProblemType from .SolutionStructs import ProblemSizes, ProblemType -from .Common import Common, print1, printExit, printWarning, assignGlobalParameters, \ +from .Common import globalParameters, print1, printExit, printWarning, assignGlobalParameters, \ restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters @@ -178,7 +178,7 @@ def TensileClientConfig(userArgs): overrideParameters = argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): print1("Overriding {0}={1}".format(key, value)) - Common.globalParameters[key] = value + globalParameters[key] = value # write output ClientWriter.writeClientConfigIni(True, sizes, "", "", "", "", conProblemType, "", [], "", args.OutputConfig, None) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 18b6b879b1..444e9fd200 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -36,10 +36,10 @@ from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles from Tensile.Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults -from Tensile.TensileInstructions import getGfxName, TensileInstructions +from Tensile.TensileInstructions import TensileInstructions from Tensile.Common import globalParameters, HR, print1, print2, printExit, IsaVersion, ensurePath, state, \ CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2, tqdm, \ - SemanticVersion + SemanticVersion, getGfxName from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H from Tensile.SolutionLibrary import MasterSolutionLibrary diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index e59a941246..426986e266 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -20,16 +20,16 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from .Formatting import __TI_DEBUG_LEVEL__, printExit +import pickle +import threading from copy import deepcopy from dataclasses import dataclass from typing import Tuple -import pickle -import subprocess -import threading -from functools import lru_cache +from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from .Formatting import __TI_DEBUG_LEVEL__, printExit + def fastdeepcopy(x): # Note: Some object can't be pickled @@ -67,10 +67,10 @@ def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool self._kernelInfo = _removeIdent(self._kernelInfo) self._kernelInfo[threading.get_ident()] = TensileInstructions.kernelInfo(isa=isaVersion) if isaVersion not in self._isaInfo: # type: ignore - asmCaps = _initAsmCaps(isaVersion, assemblerPath, debug) - archCaps = _initArchCaps(isaVersion) - regCaps = _initRegisterCaps(isaVersion, archCaps) - asmBugs = _initAsmBugs(asmCaps) + asmCaps = initAsmCaps(isaVersion, assemblerPath, debug) + archCaps = initArchCaps(isaVersion) + regCaps = initRegisterCaps(isaVersion, archCaps) + asmBugs = initAsmBugs(asmCaps) self._isaInfo[isaVersion] = TensileInstructions.IsaInfo(assemblerPath, # type: ignore asmCaps, archCaps, regCaps, asmBugs) @@ -175,11 +175,6 @@ def prettyPrint(self, indent="") -> str: ostream += str(self) return ostream -def getGfxName(arch: Tuple[int, int, int]) -> str: - # convert last digit to hex because reasons - name = str(arch[0]) + str(arch[1]) + ('%x' % arch[2]) - return 'gfx' + ''.join(map(str,name)) - def getGlcBitName(hasGLCModifier): if hasGLCModifier: return "glc" @@ -193,190 +188,4 @@ def getSlcBitName(hasGLCModifier): def _removeIdent(isaDict) -> list: ids = [th.ident for th in threading.enumerate()] isaDict = [id for id in isaDict if id in ids] - return isaDict - -def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmString: str, \ - debug: bool=False, *options) -> bool: - """ - Try to assemble the asmString for the specified target processor - Success is defined as assembler returning no error code or stderr/stdout - """ - options = list(options) - - if isaVersion[0] >= 10: - options += ['-mwavefrontsize64'] - - args = [str(assemblerPath), '-x', 'assembler', - '-target', 'amdgcn-amdhsa', - '-mcpu='+ getGfxName(isaVersion), - *options, - '-'] - - result = subprocess.run(args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = result.stdout.decode() - - if debug: - print("isaVersion: ", isaVersion) - print("asm_cmd:", ' '.join(args)) - print("asmString: ", asmString) - print("output: ", output) - print("return code: ", result.returncode) - - if output != "" or result.returncode != 0: - return False - return True - -######################################## -# Get Caps -######################################## - -@lru_cache() -def _initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: - """ Determine assembler capabilities by testing short instructions sequences """ - rv = {} - rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) - rv["HasExplicitCO"] = _tryAssembler(isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug) - rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) - - rv["HasDirectToLds"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug) - rv["HasAddLshl"] = _tryAssembler(isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug) - rv["HasLshlOr"] = _tryAssembler(isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug) - rv["HasSMulHi"] = _tryAssembler(isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug) - - rv["HasMFMA_explictB"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug) - rv["HasMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug) or rv["HasMFMA_explictB"] - rv["HasMFMA_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) or _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) - rv["HasMFMA_bf16_1k"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", isDebug) - rv["HasMFMA_f8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) - rv["HasMFMA_b8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) - - rv["HasMFMA_xf32"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", isDebug) - rv["HasSMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", isDebug) - rv["HasWMMA"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) - rv["HasWMMA_V1"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) - rv["HasWMMA_V2"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) - - rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) - - rv["v_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug) - rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) - - rv["v_pk_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug) - rv["v_pk_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug) - - rv["v_pk_add_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug) - rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug) - - rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug) - - rv["v_mad_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) - rv["v_fma_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) - - rv["v_dot2_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug) - rv["v_dot2c_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) - - rv["v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug) - rv["v_dot4c_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug) - rv["VOP3v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug) - - rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) - rv["v_fma_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug) - rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) - - rv["v_fma_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug) - - rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) - - rv["HasAtomicAdd"] = _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", isDebug) - rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", isDebug) - rv["HasMUBUFConst"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug) - rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) - - rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) - - rv["HasNTModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", isDebug) - - rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) - - if _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(63)", isDebug): - rv["MaxVmcnt"] = 63 - elif _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(15)", isDebug): - rv["MaxVmcnt"] = 15 - else: - rv["MaxVmcnt"] = 0 - - # TODO- Need to query the max cap, just like vmcnt as well? - rv["MaxLgkmcnt"] = 15 - - rv["SupportedSource"] = True - - return rv - -@lru_cache() -def _initArchCaps(isaVersion) -> dict: - rv = {} - rv["HasEccHalf"] = (isaVersion in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["Waitcnt0Disabled"] = (isaVersion in [(9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["SeparateVscnt"] = isaVersion[0] in (10, 11) - rv["SeparateLGKMcnt"] = isaVersion[0] == (12) - rv["SeparateVMcnt"] = isaVersion[0] == (12) - rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) - rv["HasWave32"] = isaVersion[0] in (10, 11, 12) - rv["HasAccCD"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["ArchAccUnifiedRegs"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["CrosslaneWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["ForceStoreSC1"] = (isaVersion in [(9,4,0), (9,4,1)]) - rv["TransOpWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) - rv["DSLow16NotPreserve"] = isaVersion[0] == (12) - rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) - rv["NoSDWA"] = isaVersion[0] == (12) - rv["VOP3ByteSel"] = isaVersion[0] == (12) - rv["HasFP8_OCP"] = isaVersion[0] == (12) - return rv - -def _initRegisterCaps(isaVersion, archCaps) -> dict: - rv = {} - rv["MaxVgpr"] = 256 - # max allowed is 112 out of 112 , 6 is used by hardware 4 SGPRs are wasted - rv["MaxSgpr"] = 102 - - rv["PhysicalMaxVgpr"] = 512 - rv["PhysicalMaxSgpr"] = 800 - - if isaVersion[0] == 10: - rv["PhysicalMaxVgprCU"] = 1024 * 32 - elif isaVersion[0] == 11: - if isaVersion[2] == 2: - rv["PhysicalMaxVgprCU"] = 1024 * 32 - else: - rv["PhysicalMaxVgprCU"] = 1536 * 32 - elif isaVersion[0] == 12: - rv["PhysicalMaxVgprCU"] = 1536 * 32 - elif isaVersion[0] == 9: - if archCaps["ArchAccUnifiedRegs"]: - rv["PhysicalMaxVgprCU"] = 2048 * 64 - else: - rv["PhysicalMaxVgprCU"] = 1024 * 64 - elif isaVersion[0] == 8: - rv["PhysicalMaxVgprCU"] = 1024 * 64 - elif isaVersion[0] == 0: - rv["PhysicalMaxVgprCU"] = 0 - else: - assert 0, "No valid VGPR value for this platform" - - return rv - -def _initAsmBugs(asmCaps) -> dict: - rv = {} - rv["ExplicitCO"] = asmCaps["HasExplicitCO"] - rv["ExplicitNC"] = asmCaps["HasExplicitNC"] - - return rv + return isaDict \ No newline at end of file diff --git a/tensilelite/Tensile/TensileInstructions/Code.py b/tensilelite/Tensile/TensileInstructions/Code.py index 808bc810d0..fbf450fa90 100644 --- a/tensilelite/Tensile/TensileInstructions/Code.py +++ b/tensilelite/Tensile/TensileInstructions/Code.py @@ -20,7 +20,8 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from .Base import Item, getGfxName +from ..Common import getGfxName +from .Base import Item from .Enums import SignatureValueKind from .Formatting import slash, slash50, block, block3Line, blockNewLine, \ formatStr, printExit diff --git a/tensilelite/Tensile/TensileInstructions/Utils.py b/tensilelite/Tensile/TensileInstructions/Utils.py index ae3620bd3d..f9cb5de4aa 100644 --- a/tensilelite/Tensile/TensileInstructions/Utils.py +++ b/tensilelite/Tensile/TensileInstructions/Utils.py @@ -20,8 +20,6 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -import warnings -from .Base import getGfxName from .Code import Module from .Containers import HolderContainer, RegisterContainer, RegName from .DataType import DataType @@ -31,10 +29,8 @@ from functools import lru_cache from math import log -from typing import List, Tuple -import os +from typing import Tuple import random -import shlex import string ######################################## diff --git a/tensilelite/Tensile/TensileRetuneLibrary.py b/tensilelite/Tensile/TensileRetuneLibrary.py index a4bed62df2..cd14afab96 100644 --- a/tensilelite/Tensile/TensileRetuneLibrary.py +++ b/tensilelite/Tensile/TensileRetuneLibrary.py @@ -27,7 +27,7 @@ from . import ClientWriter from . import LibraryIO from . import LibraryLogic -from .Common import Common, globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ +from .Common import globalParameters, print1, printWarning, ensurePath, assignGlobalParameters, \ restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters from .SolutionStructs import ProblemSizes @@ -99,7 +99,7 @@ def parseCurrentLibrary(libPath, sizePath): # get performance metric if len(libYaml) > 10: - Common.globalParameters["PerformanceMetric"] = libYaml[10] + GlobalParameters.globalParameters["PerformanceMetric"] = libYaml[10] # process exactLogic into ProblemSizes sizes = [] @@ -136,7 +136,7 @@ def runBenchmarking(solutions, problemSizes, outPath, update, cxxCompiler: str, ensurePath(resultsDir) if update: - Common.globalParameters["LibraryUpdateFile"] = os.path.join(resultsDir, "update.yaml") + globalParameters["LibraryUpdateFile"] = os.path.join(resultsDir, "update.yaml") pushWorkingPath(shortName) pushWorkingPath("source") @@ -220,7 +220,7 @@ def TensileRetuneLibrary(userArgs): overrideParameters = argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): print1("Overriding {0}={1}".format(key, value)) - Common.globalParameters[key] = value + globalParameters[key] = value # parse library logic then setup and run benchmarks (rawYaml, solutions, problemSizes) = parseCurrentLibrary(libPath, sizePath) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 0fb3adaad0..e3b9509472 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -32,8 +32,7 @@ from pathlib import Path from typing import List, Union -from ..TensileInstructions import getGfxName -from ..Common import globalParameters, print2, ensurePath, SemanticVersion +from ..Common import globalParameters, print2, ensurePath, SemanticVersion, getGfxName from ..KernelWriterAssembly import KernelWriterAssembly from ..Toolchain.Validators import getVersion from ..SolutionStructs import Solution diff --git a/tensilelite/Tensile/bin/TensileValidateLogic b/tensilelite/Tensile/bin/TensileValidateLogic new file mode 100755 index 0000000000..92cc514388 --- /dev/null +++ b/tensilelite/Tensile/bin/TensileValidateLogic @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +# This script only gets called by CMake + +try: + from Tensile import TensileValidateLogic +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + print(parentdir) + sys.path.append(parentdir) + + from Tensile import TensileValidateLogic + +################################################################################ +# Main +################################################################################ +if __name__ == "__main__": + TensileValidateLogic.run() From 8dc646aee1cd6ac5e87f5075337947c8cd73721b Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:56:59 -0600 Subject: [PATCH 026/105] fix: missing line at eof --- tensilelite/Tensile/Common/Types.py | 2 +- tensilelite/Tensile/Common/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index 0f0daccc86..8bb6eec02b 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -5,4 +5,4 @@ class SemanticVersion(NamedTuple): major: int minor: int - patch: int \ No newline at end of file + patch: int diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index 07b36c3eed..09aba3133b 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -7,4 +7,4 @@ from .Types import * # Dunder variables are not exported via `*` -from .GlobalParameters import __version__ \ No newline at end of file +from .GlobalParameters import __version__ From 364e4e533d21b0e903c4d0acbd1b9e5cbcfc5ce9 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 7 Feb 2025 15:34:09 -0600 Subject: [PATCH 027/105] fix: basic reviewer comments --- tensilelite/Tensile/ClientWriter.py | 4 +- tensilelite/Tensile/Common/Architectures.py | 73 +- tensilelite/Tensile/Common/Capabilities.py | 368 +++-- .../Tensile/Common/GlobalParameters.py | 1284 ++++++++++------- tensilelite/Tensile/Common/Parallel.py | 340 +++-- tensilelite/Tensile/Common/Types.py | 3 +- tensilelite/Tensile/Common/Utilities.py | 819 ++++++----- tensilelite/Tensile/Common/__init__.py | 10 +- tensilelite/Tensile/Contractions.py | 6 +- tensilelite/Tensile/GenerateSummations.py | 8 +- tensilelite/Tensile/Hardware.py | 8 +- .../Tensile/KernelWriterActivationFunction.py | 10 +- tensilelite/Tensile/KernelWriterConversion.py | 10 +- tensilelite/Tensile/LibraryIO.py | 6 +- tensilelite/Tensile/Ops/AMaxGenerator.py | 6 +- tensilelite/Tensile/Ops/LayerNormGenerator.py | 6 +- tensilelite/Tensile/Ops/SoftmaxGenerator.py | 10 +- tensilelite/Tensile/SolutionLibrary.py | 6 +- .../TensileCreateLibrary/ParseArguments.py | 186 ++- .../Tensile/TensileCreateLibrary/Run.py | 800 +++++----- .../Tensile/TensileCreateLibrary/__init__.py | 4 +- .../Tensile/TensileInstructions/Base.py | 2 +- .../Tensile/TensileInstructions/Code.py | 8 +- tensilelite/Tensile/Toolchain/Assembly.py | 4 +- .../Tensile/Utilities/Decorators/Profile.py | 18 +- .../Tensile/Utilities/Decorators/Shared.py | 3 +- .../Tensile/Utilities/Decorators/Timing.py | 25 +- tensilelite/Tensile/__init__.py | 7 +- tensilelite/Tensile/bin/TensileValidateLogic | 44 - .../Tensile/cmake/TensileConfigVersion.cmake | 4 +- tensilelite/tox.ini | 35 +- 31 files changed, 2394 insertions(+), 1723 deletions(-) delete mode 100755 tensilelite/Tensile/bin/TensileValidateLogic diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 428e67196d..3811861124 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -34,7 +34,7 @@ from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO -from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, getGfxName +from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex @@ -516,7 +516,7 @@ def param(key, value): libraryFile = os.path.join(sourceDir, "library", libraryFilename) param("library-file", libraryFile) - currentGFXName = getGfxName(globalParameters["CurrentISA"]) + currentGFXName = isaToGfx(globalParameters["CurrentISA"]) for coFile in codeObjectFiles: if 'gfx' not in coFile or currentGFXName in coFile: param("code-object", os.path.join(sourceDir,coFile)) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index eedcad7838..13909cce4e 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -27,29 +27,42 @@ # Translate GPU targets to filter filenames in Tensile_LOGIC directory architectureMap = { - 'all':'_', - 'gfx000':'none', - 'gfx803':'r9nano', - 'gfx900':'vega10', - 'gfx906':'vega20', 'gfx906:xnack+':'vega20', 'gfx906:xnack-':'vega20', - 'gfx908':'arcturus','gfx908:xnack+':'arcturus', 'gfx908:xnack-':'arcturus', - 'gfx90a':'aldebaran', 'gfx90a:xnack+':'aldebaran', 'gfx90a:xnack-':'aldebaran', - 'gfx940':'aquavanjaram', 'gfx940:xnack+':'aquavanjaram', 'gfx940:xnack-':'aquavanjaram', - 'gfx941':'aquavanjaram', 'gfx941:xnack+':'aquavanjaram', 'gfx941:xnack-':'aquavanjaram', - 'gfx942':'aquavanjaram', 'gfx942:xnack+':'aquavanjaram', 'gfx942:xnack-':'aquavanjaram', - 'gfx1010':'navi10', - 'gfx1011':'navi12', - 'gfx1012':'navi14', - 'gfx1030':'navi21', - 'gfx1100':'navi31', - 'gfx1101':'navi32', - 'gfx1102':'navi33', - 'gfx1200':'gfx1200', - 'gfx1201':'gfx1201', + "all": "_", + "gfx000": "none", + "gfx803": "r9nano", + "gfx900": "vega10", + "gfx906": "vega20", + "gfx906:xnack+": "vega20", + "gfx906:xnack-": "vega20", + "gfx908": "arcturus", + "gfx908:xnack+": "arcturus", + "gfx908:xnack-": "arcturus", + "gfx90a": "aldebaran", + "gfx90a:xnack+": "aldebaran", + "gfx90a:xnack-": "aldebaran", + "gfx940": "aquavanjaram", + "gfx940:xnack+": "aquavanjaram", + "gfx940:xnack-": "aquavanjaram", + "gfx941": "aquavanjaram", + "gfx941:xnack+": "aquavanjaram", + "gfx941:xnack-": "aquavanjaram", + "gfx942": "aquavanjaram", + "gfx942:xnack+": "aquavanjaram", + "gfx942:xnack-": "aquavanjaram", + "gfx1010": "navi10", + "gfx1011": "navi12", + "gfx1012": "navi14", + "gfx1030": "navi21", + "gfx1100": "navi31", + "gfx1101": "navi32", + "gfx1102": "navi33", + "gfx1200": "gfx1200", + "gfx1201": "gfx1201", } -def getGfxArch(name: str) -> Optional[IsaVersion]: - """Extracts the ISA version from a given GPU architecture name. + +def gfxToIsa(name: str) -> Optional[IsaVersion]: + """Extracts the ISA version from a given gfx architecture name. Args: name: The gfx name of the GPU architecture (e.g., 'gfx906'). @@ -58,9 +71,9 @@ def getGfxArch(name: str) -> Optional[IsaVersion]: An object representing the major, minor, and step version of the ISA. Returns None if the name does not match the expected pattern. """ - match = re.search(r'gfx([0-9a-fA-F]{3,})', name) - if not match: - return None + match = re.search(r"gfx([0-9a-fA-F]{3,})", name) + if not match: + return None ipart = match.group(1) step = int(ipart[-1], 16) @@ -71,7 +84,8 @@ def getGfxArch(name: str) -> Optional[IsaVersion]: major = int(ipart) return tuple((major, minor, step)) -def getGfxName(arch: IsaVersion) -> str: + +def isaToGfx(arch: IsaVersion) -> str: """Converts an ISA version to a gfx architecture name. Args: @@ -81,10 +95,11 @@ def getGfxName(arch: IsaVersion) -> str: The name of the GPU architecture (e.g., 'gfx906'). """ # Convert last digit to hex because reasons - name = str(arch[0]) + str(arch[1]) + ('%x' % arch[2]) - return 'gfx' + ''.join(map(str,name)) + name = str(arch[0]) + str(arch[1]) + ("%x" % arch[2]) + return "gfx" + "".join(map(str, name)) + -def getGfxCommonName(gfxName: str) -> Optional[str]: +def gfxToSwCodename(gfxName: str) -> Optional[str]: """Retrieves the common name for a given gfx architecture name. Args: @@ -100,4 +115,4 @@ def getGfxCommonName(gfxName: str) -> Optional[str]: for archKey in architectureMap: if gfxName in archKey: return architectureMap[archKey] - return None \ No newline at end of file + return None diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index f64e3cdb64..5f0977abbd 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -1,12 +1,17 @@ import subprocess - from functools import lru_cache from typing import Tuple -from .Architectures import getGfxName +from .Architectures import isaToGfx + -def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmString: str, \ - debug: bool=False, *options) -> bool: +def _tryAssembler( + isaVersion: Tuple[int, int, int], + assemblerPath: str, + asmString: str, + debug: bool = False, + *options +) -> bool: """ Try to assemble the asmString for the specified target processor Success is defined as assembler returning no error code or stderr/stdout @@ -14,20 +19,27 @@ def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmStrin options = list(options) if isaVersion[0] >= 10: - options += ['-mwavefrontsize64'] - - args = [str(assemblerPath), '-x', 'assembler', - '-target', 'amdgcn-amdhsa', - '-mcpu='+ getGfxName(isaVersion), - *options, - '-'] - - result = subprocess.run(args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + options += ["-mwavefrontsize64"] + + args = [ + str(assemblerPath), + "-x", + "assembler", + "-target", + "amdgcn-amdhsa", + "-mcpu=" + isaToGfx(isaVersion), + *options, + "-", + ] + + result = subprocess.run( + args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) output = result.stdout.decode() if debug: print("isaVersion: ", isaVersion) - print("asm_cmd:", ' '.join(args)) + print("asm_cmd:", " ".join(args)) print("asmString: ", asmString) print("output: ", output) print("return code: ", result.returncode) @@ -36,83 +48,223 @@ def _tryAssembler(isaVersion: Tuple[int, int, int], assemblerPath: str, asmStrin return False return True + ######################################## # Get Caps ######################################## + @lru_cache() def initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: - """ Determine assembler capabilities by testing short instructions sequences """ + """Determine assembler capabilities by testing short instructions sequences""" rv = {} - rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) - rv["HasExplicitCO"] = _tryAssembler(isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug) - rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) - - rv["HasDirectToLds"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug) - rv["HasAddLshl"] = _tryAssembler(isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug) - rv["HasLshlOr"] = _tryAssembler(isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug) - rv["HasSMulHi"] = _tryAssembler(isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug) - - rv["HasMFMA_explictB"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug) - rv["HasMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug) or rv["HasMFMA_explictB"] - rv["HasMFMA_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) or _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) - rv["HasMFMA_bf16_1k"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", isDebug) - rv["HasMFMA_f8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) - rv["HasMFMA_b8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) - - rv["HasMFMA_xf32"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", isDebug) - rv["HasSMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", isDebug) - rv["HasWMMA"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) - rv["HasWMMA_V1"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) - rv["HasWMMA_V2"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) - - rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) - - rv["v_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug) - rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) - - rv["v_pk_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug) - rv["v_pk_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug) - - rv["v_pk_add_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug) - rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug) - - rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug) - - rv["v_mad_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) - rv["v_fma_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) - - rv["v_dot2_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug) - rv["v_dot2c_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) - - rv["v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug) - rv["v_dot4c_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug) - rv["VOP3v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug) - - rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) - rv["v_fma_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug) - rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) - - rv["v_fma_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug) - - rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) - - rv["HasAtomicAdd"] = _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", isDebug) - rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", isDebug) - rv["HasMUBUFConst"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug) \ - or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug) - rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) - - rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) - - rv["HasNTModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", isDebug) - - rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) + rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) + rv["HasExplicitCO"] = _tryAssembler( + isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug + ) + rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) + + rv["HasDirectToLds"] = _tryAssembler( + isaVersion, + assemblerPath, + "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", + isDebug, + ) or _tryAssembler( + isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug + ) + rv["HasAddLshl"] = _tryAssembler( + isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug + ) + rv["HasLshlOr"] = _tryAssembler( + isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug + ) + rv["HasSMulHi"] = _tryAssembler( + isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug + ) + + rv["HasMFMA_explictB"] = _tryAssembler( + isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug + ) + rv["HasMFMA"] = ( + _tryAssembler( + isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug + ) + or rv["HasMFMA_explictB"] + ) + rv["HasMFMA_f64"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", + isDebug, + ) or _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", + isDebug, + ) + rv["HasMFMA_bf16_1k"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", + isDebug, + ) + rv["HasMFMA_f8"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", + isDebug, + ) + rv["HasMFMA_b8"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", + isDebug, + ) + + rv["HasMFMA_xf32"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", + isDebug, + ) + rv["HasSMFMA"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", + isDebug, + ) + rv["HasWMMA"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", + isDebug, + ) or _tryAssembler( + isaVersion, + assemblerPath, + "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", + isDebug, + ) + rv["HasWMMA_V1"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", + isDebug, + ) + rv["HasWMMA_V2"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", + isDebug, + ) + + rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) + + rv["v_fma_f16"] = _tryAssembler( + isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug + ) + rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) + + rv["v_pk_fma_f16"] = _tryAssembler( + isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug + ) + rv["v_pk_fmac_f16"] = _tryAssembler( + isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug + ) + + rv["v_pk_add_f32"] = _tryAssembler( + isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug + ) + rv["v_pk_mul_f32"] = _tryAssembler( + isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug + ) + + rv["v_pk_mul_f32"] = _tryAssembler( + isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug + ) + + rv["v_mad_mix_f32"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", + isDebug, + ) + rv["v_fma_mix_f32"] = _tryAssembler( + isaVersion, + assemblerPath, + "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", + isDebug, + ) + + rv["v_dot2_f32_f16"] = _tryAssembler( + isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug + ) + rv["v_dot2c_f32_f16"] = _tryAssembler( + isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug + ) or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) + + rv["v_dot4_i32_i8"] = _tryAssembler( + isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug + ) + rv["v_dot4c_i32_i8"] = _tryAssembler( + isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug + ) + rv["VOP3v_dot4_i32_i8"] = _tryAssembler( + isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug + ) + + rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) + rv["v_fma_f32"] = _tryAssembler( + isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug + ) + rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) + + rv["v_fma_f64"] = _tryAssembler( + isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug + ) + + rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) + + rv["HasAtomicAdd"] = _tryAssembler( + isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug + ) or _tryAssembler( + isaVersion, + assemblerPath, + "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", + isDebug, + ) + rv["HasGLCModifier"] = _tryAssembler( + isaVersion, + assemblerPath, + "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", + isDebug, + ) or _tryAssembler( + isaVersion, + assemblerPath, + "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", + isDebug, + ) + rv["HasMUBUFConst"] = _tryAssembler( + isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug + ) or _tryAssembler( + isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug + ) + rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) + + rv["HasGLCModifier"] = _tryAssembler( + isaVersion, + assemblerPath, + "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", + isDebug, + ) + + rv["HasNTModifier"] = _tryAssembler( + isaVersion, + assemblerPath, + "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", + isDebug, + ) + + rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) if _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(63)", isDebug): rv["MaxVmcnt"] = 63 @@ -128,30 +280,39 @@ def initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: return rv + @lru_cache() def initArchCaps(isaVersion) -> dict: rv = {} - rv["HasEccHalf"] = (isaVersion in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["Waitcnt0Disabled"] = (isaVersion in [(9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["SeparateVscnt"] = isaVersion[0] in (10, 11) - rv["SeparateLGKMcnt"] = isaVersion[0] == (12) - rv["SeparateVMcnt"] = isaVersion[0] == (12) - rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) - rv["HasWave32"] = isaVersion[0] in (10, 11, 12) - rv["HasAccCD"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["ArchAccUnifiedRegs"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) - rv["CrosslaneWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["ForceStoreSC1"] = (isaVersion in [(9,4,0), (9,4,1)]) - rv["TransOpWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) - rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) - rv["DSLow16NotPreserve"] = isaVersion[0] == (12) + rv["HasEccHalf"] = isaVersion in [ + (9, 0, 6), + (9, 0, 8), + (9, 0, 10), + (9, 4, 0), + (9, 4, 1), + (9, 4, 2), + ] + rv["Waitcnt0Disabled"] = isaVersion in [(9, 0, 8), (9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["SeparateVscnt"] = isaVersion[0] in (10, 11) + rv["SeparateLGKMcnt"] = isaVersion[0] == (12) + rv["SeparateVMcnt"] = isaVersion[0] == (12) + rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) + rv["HasWave32"] = isaVersion[0] in (10, 11, 12) + rv["HasAccCD"] = isaVersion in [(9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["ArchAccUnifiedRegs"] = isaVersion in [(9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["CrosslaneWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["ForceStoreSC1"] = isaVersion in [(9, 4, 0), (9, 4, 1)] + rv["TransOpWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["SDWAWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] + rv["VgprBank"] = isaVersion[0] in (10, 11, 12) + rv["DSLow16NotPreserve"] = isaVersion[0] == (12) rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) - rv["NoSDWA"] = isaVersion[0] == (12) - rv["VOP3ByteSel"] = isaVersion[0] == (12) - rv["HasFP8_OCP"] = isaVersion[0] == (12) + rv["NoSDWA"] = isaVersion[0] == (12) + rv["VOP3ByteSel"] = isaVersion[0] == (12) + rv["HasFP8_OCP"] = isaVersion[0] == (12) return rv + def initRegisterCaps(isaVersion, archCaps) -> dict: rv = {} rv["MaxVgpr"] = 256 @@ -184,9 +345,10 @@ def initRegisterCaps(isaVersion, archCaps) -> dict: return rv + def initAsmBugs(asmCaps) -> dict: rv = {} rv["ExplicitCO"] = asmCaps["HasExplicitCO"] rv["ExplicitNC"] = asmCaps["HasExplicitNC"] - return rv \ No newline at end of file + return rv diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index e4331c76fe..6556bbe377 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -25,7 +25,6 @@ import math import os.path import time - from collections import OrderedDict from copy import deepcopy @@ -46,25 +45,47 @@ ######################################## # common ######################################## -globalParameters["MinimumRequiredVersion"] = "0.0.0" # which version of tensile is required to handle all the features required by this configuration file -globalParameters["PerformanceMetric"] = "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} -globalParameters["PrintLevel"] = 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose -globalParameters["ClientLogLevel"] = 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) +globalParameters["MinimumRequiredVersion"] = ( + "0.0.0" # which version of tensile is required to handle all the features required by this configuration file +) +globalParameters["PerformanceMetric"] = ( + "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} +) +globalParameters["PrintLevel"] = ( + 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose +) +globalParameters["ClientLogLevel"] = ( + 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) +) # benchmarking -globalParameters["KernelTime"] = False # T=use device timers, F=use host timers -globalParameters["PreciseKernelTime"] = True # T=On hip, use the timestamps for kernel start and stop rather than separate events. Can provide more accurate kernel timing. For GlobalSplitU kernels, recommend disabling this to provide consistent +globalParameters["KernelTime"] = False # T=use device timers, F=use host timers +globalParameters["PreciseKernelTime"] = ( + True # T=On hip, use the timestamps for kernel start and stop rather than separate events. Can provide more accurate kernel timing. For GlobalSplitU kernels, recommend disabling this to provide consistent +) # timing between GSU / non-GSU kernels -globalParameters["CodeFromFiles"] = True # if False byte arrays will be generated during Benchmarking phase as before -globalParameters["SortProblems"] = False # sort problems by size; else use order in YAML file -globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't -globalParameters["HardwareMonitor"] = True # False: disable benchmarking client monitoring clocks using rocm-smi. -globalParameters["MinFlopsPerSync"] = 1 # Minimum number of flops per sync to increase stability for small problems -globalParameters["NumBenchmarks"] = 1 # how many benchmark data points to collect per problem/solution -globalParameters["SyncsPerBenchmark"] = 1 # how iterations of the stream synchronization for-loop to do per benchmark data point -globalParameters["EnqueuesPerSync"] = 1 # how many solution enqueues to perform per synchronization -globalParameters["MaxEnqueuesPerSync"] = -1 # max solution enqueues to perform per synchronization -globalParameters["SleepPercent"] = 300 # how long to sleep after every data point: 25 means 25% of solution time. Sleeping lets gpu cool down more. -globalParameters["SkipSlowSolutionRatio"] = 0.0 # Skip slow solution during warm-up stage. +globalParameters["CodeFromFiles"] = ( + True # if False byte arrays will be generated during Benchmarking phase as before +) +globalParameters["SortProblems"] = False # sort problems by size; else use order in YAML file +globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't +globalParameters["HardwareMonitor"] = ( + True # False: disable benchmarking client monitoring clocks using rocm-smi. +) +globalParameters["MinFlopsPerSync"] = ( + 1 # Minimum number of flops per sync to increase stability for small problems +) +globalParameters["NumBenchmarks"] = ( + 1 # how many benchmark data points to collect per problem/solution +) +globalParameters["SyncsPerBenchmark"] = ( + 1 # how iterations of the stream synchronization for-loop to do per benchmark data point +) +globalParameters["EnqueuesPerSync"] = 1 # how many solution enqueues to perform per synchronization +globalParameters["MaxEnqueuesPerSync"] = -1 # max solution enqueues to perform per synchronization +globalParameters["SleepPercent"] = ( + 300 # how long to sleep after every data point: 25 means 25% of solution time. Sleeping lets gpu cool down more. +) +globalParameters["SkipSlowSolutionRatio"] = 0.0 # Skip slow solution during warm-up stage. # The valid range of this ratio is (0.0 ~ 1.0), and 0.0 means no skipping. # Skip condition: warm-up time * ratio > current best sol's warm-up time # Suggestion: @@ -73,30 +94,54 @@ # Large size : 0.9 # cProfile -globalParameters["Profiler"] = 0 # Enable profiler. 0=off, 1=cProfile. This will set CpuThreads to 1. +globalParameters["Profiler"] = ( + 0 # Enable profiler. 0=off, 1=cProfile. This will set CpuThreads to 1. +) # validation -globalParameters["NumElementsToValidate"] = 128 # number of elements to validate, 128 will be evenly spaced out (with prime number stride) across C tensor -globalParameters["NumElementsToValidateWinner"] = 0 # number of elements to validate in LibraryClient stage, the exact number to be validated is max(NumElementsToValidate,NumElementsToValidateWinner) -globalParameters["BoundsCheck"] = 0 # Bounds check -#1: Perform bounds check to find out of bounds reads/writes. NumElementsToValidate must be -1. -#2: Perform bounds check by front side guard page -#3: Perform bounds check by back side guard page -#4: Perform bounds check by both back and front side guard page - -globalParameters["ValidationMaxToPrint"] = 4 # maximum number of mismatches to print -globalParameters["ValidationPrintValids"] = False # print matches too +globalParameters["NumElementsToValidate"] = ( + 128 # number of elements to validate, 128 will be evenly spaced out (with prime number stride) across C tensor +) +globalParameters["NumElementsToValidateWinner"] = ( + 0 # number of elements to validate in LibraryClient stage, the exact number to be validated is max(NumElementsToValidate,NumElementsToValidateWinner) +) +globalParameters["BoundsCheck"] = 0 # Bounds check +# 1: Perform bounds check to find out of bounds reads/writes. NumElementsToValidate must be -1. +# 2: Perform bounds check by front side guard page +# 3: Perform bounds check by back side guard page +# 4: Perform bounds check by both back and front side guard page + +globalParameters["ValidationMaxToPrint"] = 4 # maximum number of mismatches to print +globalParameters["ValidationPrintValids"] = False # print matches too # steps -globalParameters["ForceRedoBenchmarkProblems"] = True # if False and benchmarking already complete, then benchmarking will be skipped when tensile is re-run -globalParameters["ForceRedoLibraryLogic"] = True # if False and library logic already analyzed, then library logic will be skipped when tensile is re-run -globalParameters["ForceRedoLibraryClient"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run - -globalParameters["ShowProgressBar"] = True # if False and library client already built, then building library client will be skipped when tensile is re-run -globalParameters["SolutionSelectionAlg"] = 1 # algorithm to determine which solutions to keep. 0=removeLeastImportantSolutions, 1=keepWinnerSolutions (faster) -globalParameters["ExpandRanges"] = True # expand ranges into exact configs before writing logic file. False ignores ranges. -globalParameters["GenerateSourcesAndExit"] = False # Exit after kernel source generation. -globalParameters["WavefrontWidth"] = 64 # if False and library client already built, then building library client will be skipped when tensile is re-run -globalParameters["ExitOnFails"] = 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. -globalParameters["CpuThreads"] = -1 # How many CPU threads to use for kernel generation. 0=no threading, -1 == nproc, N=min(nproc,N). TODO - 0 sometimes fails with a kernel name error? 0 does not check error codes correctly +globalParameters["ForceRedoBenchmarkProblems"] = ( + True # if False and benchmarking already complete, then benchmarking will be skipped when tensile is re-run +) +globalParameters["ForceRedoLibraryLogic"] = ( + True # if False and library logic already analyzed, then library logic will be skipped when tensile is re-run +) +globalParameters["ForceRedoLibraryClient"] = ( + True # if False and library client already built, then building library client will be skipped when tensile is re-run +) + +globalParameters["ShowProgressBar"] = ( + True # if False and library client already built, then building library client will be skipped when tensile is re-run +) +globalParameters["SolutionSelectionAlg"] = ( + 1 # algorithm to determine which solutions to keep. 0=removeLeastImportantSolutions, 1=keepWinnerSolutions (faster) +) +globalParameters["ExpandRanges"] = ( + True # expand ranges into exact configs before writing logic file. False ignores ranges. +) +globalParameters["GenerateSourcesAndExit"] = False # Exit after kernel source generation. +globalParameters["WavefrontWidth"] = ( + 64 # if False and library client already built, then building library client will be skipped when tensile is re-run +) +globalParameters["ExitOnFails"] = ( + 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. +) +globalParameters["CpuThreads"] = ( + -1 +) # How many CPU threads to use for kernel generation. 0=no threading, -1 == nproc, N=min(nproc,N). TODO - 0 sometimes fails with a kernel name error? 0 does not check error codes correctly globalParameters["NumWarmups"] = 0 # even if error occurs in kernel generation (ie due to resource overflow), @@ -108,15 +153,21 @@ # optimization knob controls ######################################## -globalParameters["UnrollLoopEfficiencyEnable"] = False # if True split(S) MAC&LDS in each unroll iteration into n smaller groups.. +globalParameters["UnrollLoopEfficiencyEnable"] = ( + False # if True split(S) MAC&LDS in each unroll iteration into n smaller groups.. +) ######################################## # less common ######################################## -globalParameters["CMakeBuildType"] = "Release" # whether benchmark clients and library client should be release or debug -globalParameters["PrintSolutionRejectionReason"] = False # when a solution is marked as invalid, print why -globalParameters["LogicFormat"] = "yaml" # set library backend (yaml, or json) -globalParameters["LibraryFormat"] = "yaml" # set library backend (yaml, or msgpack) +globalParameters["CMakeBuildType"] = ( + "Release" # whether benchmark clients and library client should be release or debug +) +globalParameters["PrintSolutionRejectionReason"] = ( + False # when a solution is marked as invalid, print why +) +globalParameters["LogicFormat"] = "yaml" # set library backend (yaml, or json) +globalParameters["LibraryFormat"] = "yaml" # set library backend (yaml, or msgpack) # True/False: CSV will/won't export WinnerGFlops, WinnerTimeUS, WinnerIdx, WinnerName. # TODO - if no side-effect, we can set default to True. This can make analyzing "LibraryLogic" (AddFromCSV) faster @@ -144,11 +195,11 @@ # For Alpha/Beta: Only problem-independent init (0~7, 16, 23~26) can be used, # problem-dependent init (8~15) would cause a exception (Invalid InitMode) in New Client globalParameters["DataInitTypeAB"] = 3 -globalParameters["DataInitTypeA"] = -1 -globalParameters["DataInitTypeB"] = -1 -globalParameters["DataInitTypeC"] = 3 -globalParameters["DataInitTypeD"] = 0 -globalParameters["DataInitTypeE"] = 0 +globalParameters["DataInitTypeA"] = -1 +globalParameters["DataInitTypeB"] = -1 +globalParameters["DataInitTypeC"] = 3 +globalParameters["DataInitTypeD"] = 0 +globalParameters["DataInitTypeE"] = 0 globalParameters["DataInitTypeAlpha"] = 2 globalParameters["DataInitTypeBeta"] = 2 globalParameters["DataInitTypeBias"] = 3 @@ -158,79 +209,133 @@ globalParameters["DataInitTypeScaleD"] = 2 globalParameters["DataInitTypeScaleAlphaVec"] = 3 globalParameters["DataInitValueActivationArgs"] = [2.0, 2.0] -globalParameters["CEqualD"] = False # Set to true if testing for the case where the pointer to C is the same as D. +globalParameters["CEqualD"] = ( + False # Set to true if testing for the case where the pointer to C is the same as D. +) # When this parameter is set to 0, the Tensile client will use srand(time(NULL)). # If not 0 the Tensile client will use srand(seed). globalParameters["DataInitSeed"] = 0 -globalParameters["PruneSparseMode"] = 0 # Prune mode for Sparse Matrix: 0=random, 1=XX00, 2=X0X0, 3=0XX0, 4=X00X, 5=0X0X, 6=00XX +globalParameters["PruneSparseMode"] = ( + 0 # Prune mode for Sparse Matrix: 0=random, 1=XX00, 2=X0X0, 3=0XX0, 4=X00X, 5=0X0X, 6=00XX +) # build parameters -globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake -globalParameters["CMakeCFlags"] = "" # pass flags to cmake -globalParameters["DebugKernel"] = False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed -globalParameters["AsanBuild"] = False # build with asan -globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels -globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp +globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake +globalParameters["CMakeCFlags"] = "" # pass flags to cmake +globalParameters["DebugKernel"] = ( + False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed +) +globalParameters["AsanBuild"] = False # build with asan +globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels +globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp # debug for assembly -globalParameters["EnableAsserts"] = False # Enable assembly debug assert -globalParameters["EnableDebugA"] = False # Enable / Disable CheckValue1A -globalParameters["EnableDebugB"] = False # Enable / Disable CheckValue1B -globalParameters["EnableDebugC"] = False # Enable / Disable CheckValueC -globalParameters["ExpectedValueC"] = 16.0 # Expected C Value when CheckValueC, debug for Alpha*A*B -globalParameters["ForceCExpectedValue"] = False # Force C to "DebugExpectedValueC", debug for global write -globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM +globalParameters["EnableAsserts"] = False # Enable assembly debug assert +globalParameters["EnableDebugA"] = False # Enable / Disable CheckValue1A +globalParameters["EnableDebugB"] = False # Enable / Disable CheckValue1B +globalParameters["EnableDebugC"] = False # Enable / Disable CheckValueC +globalParameters["ExpectedValueC"] = 16.0 # Expected C Value when CheckValueC, debug for Alpha*A*B +globalParameters["ForceCExpectedValue"] = ( + False # Force C to "DebugExpectedValueC", debug for global write +) +globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM # Tensor printing controls: -globalParameters["PrintTensorA"] = 0 # Print TensorA after initialization -globalParameters["PrintTensorB"] = 0 # Print TensorB after initialization -globalParameters["PrintTensorC"] = 0 # Print TensorC. 0x1=after init; 0x2=after copy-back; 0x3=both -globalParameters["PrintTensorD"] = 0 # Print TensorD. 0x1=after init; 0x2=after copy-back; 0x3=both -globalParameters["PrintTensorRef"] = 0 # Print reference tensor. 0x1=after init; 0x2=after copy-back; 0x3=both -globalParameters["PrintTensorBias"] = 0 # Print TensorBias after initialization -globalParameters["PrintTensorAmaxD"] = 0 # Print AmaxD after validation -globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info -globalParameters["PrintWinnersOnly"] = False # Only print the solutions which become the fastest -globalParameters["PrintCodeCommands"] = False # print the commands used to generate the code objects (asm,link,hip-clang, etc) -globalParameters["DumpTensors"] = False # If True, dump tensors to binary files instead of printing them. +globalParameters["PrintTensorA"] = 0 # Print TensorA after initialization +globalParameters["PrintTensorB"] = 0 # Print TensorB after initialization +globalParameters["PrintTensorC"] = ( + 0 # Print TensorC. 0x1=after init; 0x2=after copy-back; 0x3=both +) +globalParameters["PrintTensorD"] = ( + 0 # Print TensorD. 0x1=after init; 0x2=after copy-back; 0x3=both +) +globalParameters["PrintTensorRef"] = ( + 0 # Print reference tensor. 0x1=after init; 0x2=after copy-back; 0x3=both +) +globalParameters["PrintTensorBias"] = 0 # Print TensorBias after initialization +globalParameters["PrintTensorAmaxD"] = 0 # Print AmaxD after validation +globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info +globalParameters["PrintWinnersOnly"] = False # Only print the solutions which become the fastest +globalParameters["PrintCodeCommands"] = ( + False # print the commands used to generate the code objects (asm,link,hip-clang, etc) +) +globalParameters["DumpTensors"] = ( + False # If True, dump tensors to binary files instead of printing them. +) # If PrintMax* is greater than the dimension, the middle elements will be replaced with "..." # device selection -globalParameters["Platform"] = 0 # select opencl platform -globalParameters["Device"] = 0 # select hip device or opencl device within platform +globalParameters["Platform"] = 0 # select opencl platform +globalParameters["Device"] = 0 # select hip device or opencl device within platform # shouldn't need to change -globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy -globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use -globalParameters["ShortNames"] = False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids - -globalParameters["MaxFileName"] = 64 # If a file name would be longer than this, shorten it with a hash. -globalParameters["SupportedISA"] = [(8,0,3), (9,0,0), (9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2), (10,1,0), (10,1,1), (10,1,2), (10,3,0), (11,0,0), (11,0,1), (11,0,2), (12,0,0), (12,0,1)] # assembly kernels writer supports these architectures - -globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. -globalParameters["ClientBuildPath"] = "0_Build" # subdirectory for host code build directory -globalParameters["BenchmarkProblemsPath"] = "1_BenchmarkProblems" # subdirectory for benchmarking phases -globalParameters["BenchmarkDataPath"] = "2_BenchmarkData" # subdirectory for storing final benchmarking data -globalParameters["LibraryLogicPath"] = "3_LibraryLogic" # subdirectory for library logic produced by analysis -globalParameters["LibraryClientPath"] = "4_LibraryClient" # subdirectory for building example library client -globalParameters["ClientExecutionLockPath"] = None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. -globalParameters["LibraryUpdateFile"] = "" # File name for writing indices and speeds suitable for updating an existing library logic file -globalParameters["LibraryUpdateComment"] = False # Include solution name as a comment in the library update file +globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy +globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use +globalParameters["ShortNames"] = ( + False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids +) + +globalParameters["MaxFileName"] = ( + 64 # If a file name would be longer than this, shorten it with a hash. +) +globalParameters["SupportedISA"] = [ + (8, 0, 3), + (9, 0, 0), + (9, 0, 6), + (9, 0, 8), + (9, 0, 10), + (9, 4, 0), + (9, 4, 1), + (9, 4, 2), + (10, 1, 0), + (10, 1, 1), + (10, 1, 2), + (10, 3, 0), + (11, 0, 0), + (11, 0, 1), + (11, 0, 2), + (12, 0, 0), + (12, 0, 1), +] # assembly kernels writer supports these architectures + +globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. +globalParameters["ClientBuildPath"] = "0_Build" # subdirectory for host code build directory +globalParameters["BenchmarkProblemsPath"] = ( + "1_BenchmarkProblems" # subdirectory for benchmarking phases +) +globalParameters["BenchmarkDataPath"] = ( + "2_BenchmarkData" # subdirectory for storing final benchmarking data +) +globalParameters["LibraryLogicPath"] = ( + "3_LibraryLogic" # subdirectory for library logic produced by analysis +) +globalParameters["LibraryClientPath"] = ( + "4_LibraryClient" # subdirectory for building example library client +) +globalParameters["ClientExecutionLockPath"] = ( + None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. +) +globalParameters["LibraryUpdateFile"] = ( + "" # File name for writing indices and speeds suitable for updating an existing library logic file +) +globalParameters["LibraryUpdateComment"] = ( + False # Include solution name as a comment in the library update file +) # internal, i.e., gets set during startup -globalParameters["CurrentISA"] = (0,0,0) -globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch -globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator -globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi +globalParameters["CurrentISA"] = (0, 0, 0) +globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch +globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator +globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi globalParameters["HipClangVersion"] = "0.0.0" # default runtime is selected based on operating system, user can override if os.name == "nt": - globalParameters["RuntimeLanguage"] = "HIP" #"OCL" + globalParameters["RuntimeLanguage"] = "HIP" # "OCL" else: - globalParameters["RuntimeLanguage"] = "HIP" + globalParameters["RuntimeLanguage"] = "HIP" globalParameters["CodeObjectVersion"] = "4" globalParameters["Architecture"] = "all" @@ -246,153 +351,218 @@ globalParameters["PerfModelReadEfficiency"] = 0.85 # limitation for training -globalParameters["MaxWorkspaceSize"] = 128 * 1024 * 1024 # max workspace for training (128MB) -globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) +globalParameters["MaxWorkspaceSize"] = 128 * 1024 * 1024 # max workspace for training (128MB) +globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) # control if a solution is run for a given problem globalParameters["GranularityThreshold"] = 0.0 -globalParameters["PristineOnGPU"] = True # use Pristine memory on Tensile trainning verification or not +globalParameters["PristineOnGPU"] = ( + True # use Pristine memory on Tensile trainning verification or not +) -globalParameters["SeparateArchitectures"] = False # write Tensile library metadata to separate files for each architecture +globalParameters["SeparateArchitectures"] = ( + False # write Tensile library metadata to separate files for each architecture +) -globalParameters["LazyLibraryLoading"] = False # Load library and code object files when needed instead of at startup +globalParameters["LazyLibraryLoading"] = ( + False # Load library and code object files when needed instead of at startup +) -globalParameters["EnableMarker"] = False # Enable Tensile markers +globalParameters["EnableMarker"] = False # Enable Tensile markers globalParameters["UseUserArgs"] = False -globalParameters["RotatingBufferSize"] = 0 # Size in MB -globalParameters["RotatingMode"] = 0 # Default is 0, allocated in order A0B0C0D0..ANBNCNDN. 1 is in order A0 pad B0 pad .... AN pad BN pad. - # Mode 0 requires memcpy everytime when the problem changes to reset the data, but mode 1 doesn't. +globalParameters["RotatingBufferSize"] = 0 # Size in MB +globalParameters["RotatingMode"] = ( + 0 # Default is 0, allocated in order A0B0C0D0..ANBNCNDN. 1 is in order A0 pad B0 pad .... AN pad BN pad. +) +# Mode 0 requires memcpy everytime when the problem changes to reset the data, but mode 1 doesn't. globalParameters["BuildIdKind"] = "sha1" globalParameters["ValidateLibrary"] = False -globalParameters["AsmDebug"] = False # Set to True to keep debug information for compiled code objects +globalParameters["AsmDebug"] = ( + False # Set to True to keep debug information for compiled code objects +) -globalParameters["UseEffLike"] = True # Set to False to use winnerGFlops as the performance metric +globalParameters["UseEffLike"] = True # Set to False to use winnerGFlops as the performance metric # Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen defaultGlobalParameters = deepcopy(globalParameters) - - ################################################################################ # Tensile internal parameters ################################################################################ # These parameters are not adjustable by the config yamls. They change with the # generator versions internalParameters = { - # Each universal kernel will generate one PostGSU(GlobalSplitUPGR) kernel - "GlobalSplitUPGR": 16 + # Each universal kernel will generate one PostGSU(GlobalSplitUPGR) kernel + "GlobalSplitUPGR": 16 } # These parameters are used in ContractionSolutions for user arguments support. defaultInternalSupportParams = { - "KernArgsVersion": 2, - # Information about user input internal kernel argument support - # Change this to False if the CustomKernel does not support. - "SupportUserGSU": True, - # This is a little different from GSU because GSU is already a parameter, - # but WGM is not. - "SupportCustomWGM": True, - "SupportCustomStaggerU": True, - # Use GG as G's backend - "UseUniversalArgs": True + "KernArgsVersion": 2, + # Information about user input internal kernel argument support + # Change this to False if the CustomKernel does not support. + "SupportUserGSU": True, + # This is a little different from GSU because GSU is already a parameter, + # but WGM is not. + "SupportCustomWGM": True, + "SupportCustomStaggerU": True, + # Use GG as G's backend + "UseUniversalArgs": True, } - - ################################################################################ # Enumerate Valid Solution Parameters ################################################################################ validWorkGroups = [] for numThreads in range(32, 1025, 32): - for nsg in [ 1, 2, 4, 8, 16, 32, 64, 96, 128, 256 ]: - for sg0 in range(1, numThreads//nsg+1): - sg1 = numThreads//nsg//sg0 - if sg0*sg1*nsg == numThreads: - workGroup = [sg0, sg1, nsg] - validWorkGroups.append(workGroup) - -validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list(range(20, 256, 4)) + for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: + for sg0 in range(1, numThreads // nsg + 1): + sg1 = numThreads // nsg // sg0 + if sg0 * sg1 * nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + +validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list( + range(20, 256, 4) +) validThreadTiles = [] for i in validThreadTileSides: - for j in validThreadTileSides: - validThreadTiles.append([i, j]) - -validActivationFormats = ('NCHW', 'NHWC', 'CNHW', 'NCDHW', 'NDHWC', 'CNDHW') -validWeightFormats = ('KCYX', "KYXC", "CKYX", "CYXK", 'KCZYX', 'CKZYX', 'CZYXK') -validMacroTileSides = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 6, 12, 24, 48, 96, 192, 384, 768 ] + for j in validThreadTileSides: + validThreadTiles.append([i, j]) + +validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW") +validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK") +validMacroTileSides = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 6, + 12, + 24, + 48, + 96, + 192, + 384, + 768, +] validMacroTiles = [] -validISA = [(0,0,0)] +validISA = [(0, 0, 0)] validISA.extend(globalParameters["SupportedISA"]) -depthUs = list(range(2,1024+1,1)) +depthUs = list(range(2, 1024 + 1, 1)) for i in validMacroTileSides: - for j in validMacroTileSides: - validMacroTiles.append([i, j]) + for j in validMacroTileSides: + validMacroTiles.append([i, j]) validMFMA = {} -validMFMA["H"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] -validMFMA["S"] = [[32,32,1,2], [32,32,2,1], [16,16,1,4], [16,16,4,1], [4,4,1,16]] -validMFMA["B"] = [[32,32,2,2], [32,32,4,1], [16,16,2,4], [16,16,8,1], [4,4,2,16]] -validMFMA["4xi8"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16], [32,32,16,1], [16,16,32,1]] -validMFMA["D"] = [[16,16,4,1], [4,4,4,4]] -validMFMA["B1k"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] +validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] +validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] +validMFMA["4xi8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], + [32, 32, 16, 1], + [16, 16, 32, 1], +] +validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] +validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] validMFMA["C"] = validMFMA["S"] validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [[32,32,4,2], [32,32,8,1], [16,16,4,4], [16,16,16,1], [4,4,4,16]] + [[32,32,16,1], [16,16,32,1]] -validMFMA["X"] = [[32,32,4,1], [16,16,8,1]] -validMFMA["F8"] = [[32,32,16,1], [16,16,32,1]] +validMFMA["I8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], +] + [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] +validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] validMFMA["B8"] = validMFMA["F8"] validMFMA["F8B8"] = validMFMA["F8"] validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32,32,16,1], [16,16,32,1]] +validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] validMFMA["B8N"] = validMFMA["F8N"] validMFMA["F8B8N"] = validMFMA["F8N"] validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [[16,16,16,1], ] +validWMMA = [ + [16, 16, 16, 1], +] validTT = 32 validMFMA["_format9"] = [] -for MFMA in [validMFMA["H"], validMFMA["S"], validMFMA["B"], validMFMA["D"], validMFMA["X"], validMFMA["F8N"], validWMMA]: - for MI in MFMA: - for bm in range(int(math.log(MI[3],2))+1): - for tt0 in range(1,validTT+1): - for tt1 in range(1,validTT+1): - for wave_m in range (3): - for wave_n in range(3): - validMFMA["_format9"].append([MI[0],MI[1],MI[2],MI[3],2**bm,tt0,tt1,2**wave_m, 2**wave_n]) -validMatrixInstructions = [[], [-1]] + validMFMA["H"] + validMFMA["S"] + validMFMA["B"] + validMFMA["D"] + validMFMA["B1k"] + validMFMA["X"] +for MFMA in [ + validMFMA["H"], + validMFMA["S"], + validMFMA["B"], + validMFMA["D"], + validMFMA["X"], + validMFMA["F8N"], + validWMMA, +]: + for MI in MFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validMatrixInstructions = ( + [[], [-1]] + + validMFMA["H"] + + validMFMA["S"] + + validMFMA["B"] + + validMFMA["D"] + + validMFMA["B1k"] + + validMFMA["X"] +) validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] validSMFMA = {} -validSMFMA["H"] = [[32,32,16,1], [16,16,32,1]] -validSMFMA["B"] = [[32,32,16,1], [16,16,32,1]] -validSMFMA["4xi8"] = [[32,32,32,1], [16,16,64,1]] +validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32,32,32,1], [16,16,64,1]] +validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] validSMFMA["B8"] = validSMFMA["F8"] validSMFMA["F8B8"] = validSMFMA["F8"] validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32,32,32,1], [16,16,64,1]] +validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] validSMFMA["B8N"] = validSMFMA["F8N"] validSMFMA["F8B8N"] = validSMFMA["F8N"] validSMFMA["B8F8N"] = validSMFMA["F8N"] validSMFMA["_format9"] = [] for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3],2))+1): - for tt0 in range(1,validTT+1): - for tt1 in range(1,validTT+1): - for wave_m in range (3): - for wave_n in range(3): - validSMFMA["_format9"].append([MI[0],MI[1],MI[2],MI[3],2**bm,tt0,tt1,2**wave_m, 2**wave_n]) + for MI in SMFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validSMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] - +validMatrixInstructions = ( + validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] +) # The supported typed GEMM, each entry is (Ti, To, Tc). @@ -402,50 +572,122 @@ # Cinternal: basically should == ComputeDataType # This is used in SolutionStruct.py::checkIfSupportedGEMMType() -validGEMMTypes = [ ('H','H','H'), ('S','S','S'), ('D','D','D'), ('C','C','C'), ('Z','Z','Z'), \ - ('H','H','S'), ('H','S','S'), \ - ('B','B','S'), ('B','S','S'), ('B','H','S'), \ - ('I8','I','I'), ('4xi8','I','I'), ('I8','I8','I'), \ - ('I8','I','S'), ('I8','I8','S'), ('I8', 'H', 'S'), ('I8', 'B', 'S'), \ - ('F8','S','S'), ('B8','S','S'), \ - ('F8B8','S','S'), ('B8F8', 'S', 'S'), \ - ('F8','H','S'), ('B8','H','S'), \ - ('F8B8','H','S'), ('B8F8','H','S'), ('B8','B','S'), \ - ('H','F8','S'), ('F8','B','S'), ('F8B8','B','S'), ('B8F8','B','S'), \ - # in/out are both R8 - ('F8','F8','S'), ('B8','B8','S'), \ - ('F8B8','B8','S'), ('B8F8', 'B8', 'S'), \ - ('F8','B8','S'), ('B8','F8','S'), \ - ('F8B8','F8','S'), ('B8F8', 'F8', 'S'), \ - # F8 NANOO - ('F8N','S','S'), ('B8N','S','S'), \ - ('F8B8N','S','S'), ('B8F8N', 'S', 'S'), \ - ('F8N','H','S'), ('B8N','H','S'), \ - ('F8B8N','H','S'), ('B8F8N','H','S'), ('B8N','B','S'), \ - ('H','F8N','S'), ('F8N','B','S'), ('F8B8N','B','S'), ('B8F8N','B','S'), \ - # in/out are both R8 - ('F8N','F8N','S'), ('B8N','B8N','S'), \ - ('F8B8N','B8N','S'), ('B8F8N', 'B8N', 'S'), \ - ('F8N','B8N','S'), ('B8N','F8N','S'), \ - ('F8B8N','F8N','S'), ('B8F8N', 'F8N', 'S') ] +validGEMMTypes = [ + ("H", "H", "H"), + ("S", "S", "S"), + ("D", "D", "D"), + ("C", "C", "C"), + ("Z", "Z", "Z"), + ("H", "H", "S"), + ("H", "S", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I8", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("B8", "B", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), + ("B8F8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), # F8 NANOO + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("B8N", "B", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), + ("B8F8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] # All HPA types are listed here (HPA=T). The name of the library logic files for these types is: # *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. # The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. -HPATypes = [ ('H','S','S'), ('H','H','S'), ('B','B','S'), ('B','S','S'), ('B','H','S'), ('I8','I','I'), \ - ('4xi8','I','I'), ('I8','I','S'), ('I8','I8','S'), ('I8', 'H', 'S'), ('I8', 'B', 'S'),\ - ('F8','S','S'), ('B8','S','S'), ('F8B8','S','S'), ('B8F8', 'S', 'S'), \ - ('F8','H','S'), ('B8','H','S'), ('F8B8','H','S'), ('B8F8','H','S'), \ - ('H','F8','S'), ('F8','B','S'), ('F8B8','B','S'), \ - # in/out are both R8 - ('F8','F8','S'), ('B8','B8','S'), ('F8B8','B8','S'), ('B8F8', 'B8', 'S'), \ - ('F8','B8','S'), ('B8','F8','S'), ('F8B8','F8','S'), ('B8F8', 'F8', 'S'), \ - ('F8N','S','S'), ('B8N','S','S'), ('F8B8N','S','S'), ('B8F8N', 'S', 'S'), \ - ('F8N','H','S'), ('B8N','H','S'), ('F8B8N','H','S'), ('B8F8N','H','S'), \ - ('H','F8N','S'), ('F8N','B','S'), ('F8B8N','B','S'), \ - # in/out are both R8 - ('F8N','F8N','S'), ('B8N','B8N','S'), ('F8B8N','B8N','S'), ('B8F8N', 'B8N', 'S'), \ - ('F8N','B8N','S'), ('B8N','F8N','S'), ('F8B8N','F8N','S'), ('B8F8N', 'F8N', 'S') ] +HPATypes = [ + ("H", "S", "S"), + ("H", "H", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] validParameters = { # 0: Global read is along parallel direction in thread level, @@ -478,13 +720,11 @@ # | [t31] [t31] [t63] [t63] # V [load_1] [load_2] [load_1] [load_2] # - "WaveSeparateGlobalReadA": [ 0, 1, 2 ], - "WaveSeparateGlobalReadB": [ 0, 1, 2 ], - + "WaveSeparateGlobalReadA": [0, 1, 2], + "WaveSeparateGlobalReadB": [0, 1, 2], # Add an unrolled loop and NGLL loop with swapped GRA and GRB order. # which may change the tlb thrashing behavior. "UnrollLoopSwapGlobalReadOrder": [0, 1], - # PrefetchGlobalRead = 1: # Requires 2X LDS space, and VGPRs for buffering data on way into LDS # prefetch / double-buffer reads from global memory -> vgprs -> lds. @@ -493,16 +733,13 @@ # Do another prefetch while writing data from vgpr to lds. # prefetch / double-buffer reads from global memory -> vgprs --> lds. # |-> prefetch reads - "PrefetchGlobalRead": [ 0, 1, 2 ], - + "PrefetchGlobalRead": [0, 1, 2], # number of iteration prefetch local reads from lds to VGPRs buffer = PLR - "PrefetchLocalRead": list(range(128+1)), - + "PrefetchLocalRead": list(range(128 + 1)), # MatrixInstruction Only # If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead # So we can schedule these localReads to the front of the loop - "ClusterLocalRead": [0,1], - + "ClusterLocalRead": [0, 1], # We use double LDS buffer when PrefetchGlobalRead. # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. @@ -514,43 +751,35 @@ # SIA2: 1LDSBuffer is set to 1 natively # SIA3: 1LDSBuffer works only when PGR=True # TODO: optimize scheduling to support more cases. - "1LDSBuffer": [-1 ,0, 1], - + "1LDSBuffer": [-1, 0, 1], # Split the unroll summation into multiple sections and combine the sections # GSU applies only to the unroll summation dimension # Set to 0 to disable GSU, kernel code will be generated without GSU support - "GlobalSplitU": list(range(0, 1024+1)), - + "GlobalSplitU": list(range(0, 1024 + 1)), # choose how to do GlobalSplitU # 1: use atomic operation to accumulate on one buffer # 2: each GSU group write to each own buffer and accumulate by another kernel # 3: each GSU group write to each own buffer and accumulate by same kernel - "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], - + "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], # don't create a whole copy of the Unroll loop with loads removed - instead # use buffer limits to suppress global loads and ignore unnecessary ds_reads - "SuppressNoLoadLoop": [False, True], - + "SuppressNoLoadLoop": [False, True], # For PrefetchGlobalRead=1, create a second copy of the unroll loop with # the LDS pointer swaps expanded into inline constants for LDS read and write instructions # This eliminates 4 vector XOR instructions used for pointer swap - "ExpandPointerSwap": [False, True], - + "ExpandPointerSwap": [False, True], # Schedule global reads and global read increments into LocalRead iterations # Can reduce pressure on local read instruction dispatch queue # 0=perform global reads at start of instruction loop # 1=schedule into the local read instruction iterations - "ScheduleGlobalRead": [0, 1], - + "ScheduleGlobalRead": [0, 1], # Schedule local writes into LocalRead iterations. # Can reduce pressure on local read instruction dispatch queue - "ScheduleLocalWrite": [0, 1], - + "ScheduleLocalWrite": [0, 1], # Scheduling algorithm to use for each iteration: # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, # followed by local writes, followed by MACs - "ScheduleIterAlg": [0, 1, 2, 3], - + "ScheduleIterAlg": [0, 1, 2, 3], # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma # the purpose of this parameter is to control density of global read instruction scheduling # Scheduling global read back to back can have better memory efficiency @@ -558,7 +787,7 @@ # Range from 0.01 to 32 # 0.1 means 1 GR per 10 mfma # 5 means 5 GR per 1 mfma - "GlobalReadPerMfma": [ i/100 for i in range(1,3200)], + "GlobalReadPerMfma": [i / 100 for i in range(1, 3200)], # # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma # the purpose of this parameter is to control density of local write instruction scheduling @@ -572,28 +801,23 @@ # 5 means 5 LW per 1 mfma # -1 will derived an optimized value internally # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) - "LocalWritePerMfma": [ i/100 for i in range(1,3200)] + [ -1 ], - + "LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1], # Interleave alpha scale calculation with beta loads and address calcs - rather # than as a separate block of instructions - "InterleaveAlpha": [0, 1], - + "InterleaveAlpha": [0, 1], # Create a copy of NoLoadLoop which interleaves the stores with the final mac # calculation and may perform other optimizations # 0 = no interleave # 1 = interleave one stores after required macs have completed execution # 2 = interleave two stores after required macs have completed execution - "OptNoLoadLoop": [0, 1, 2], - - "BufferLoad": [ False, True ], - "BufferStore": [ False, True ], - + "OptNoLoadLoop": [0, 1, 2], + "BufferLoad": [False, True], + "BufferStore": [False, True], # Attempt to load directly from global memory into Vgpr. # Assembly only - "DirectToVgprA": [ False, True ], - "DirectToVgprB": [ False, True ], - "DirectToVgprSparseMetadata": [ False, True ], - + "DirectToVgprA": [False, True], + "DirectToVgprB": [False, True], + "DirectToVgprSparseMetadata": [False, True], # Attempt to load directly from global memory into LDS. # Assembly only # Requires BufferLoad, assembler support for lds modifier on buffer @@ -607,8 +831,7 @@ # GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now) # TransposeLDS = 1 for TLU=0 case # DirectToLds support for x1 only for now - "DirectToLds": [ False, True ], - + "DirectToLds": [False, True], # Load options: # (GRO = Global Read Offset) # BufferLoad=0: @@ -644,27 +867,21 @@ # - Requirements for UseInstOffsetForGRO=1: # - BufferLoad=1 # - DirectToLds=1 - # converting m0 update from LocalWriteAddrSGpr using is usually win # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseInstOffsetForGRO": [ -1, 0, 1], - - + "UseInstOffsetForGRO": [-1, 0, 1], # Converting VGPR GRO into SGPR GRO is usually a win # However, the mode may exhaust all available SGPR, in particular for large unroll # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseSgprForGRO": [ -1, 0, 1], - + "UseSgprForGRO": [-1, 0, 1], # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes - "Use64bShadowLimit": [ True, False], - + "Use64bShadowLimit": [True, False], # Assertion properties # These provide information or assertions that the problem size meets certain requirements # for sizes or alignments. The kernel generator can use this information to produce # a kernel which uses those assertions to produce a faster kernel. # # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h - # Kernel generator will assume that the summation size is some multiple of the element size # and uses this to optimize the kernel. # This can result in more efficient kernels, but requires runtime checking to ensure the specified @@ -679,8 +896,7 @@ # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 # # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertSummationElementMultiple": [1,2,4,8,16,32,64,128], - + "AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128], # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size # and uses this to optimize the kernel. # FreeIndex[0] is usually letter "I" @@ -698,25 +914,20 @@ # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) # # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree0ElementMultiple" : [1,2,4,8,16], - + "AssertFree0ElementMultiple": [1, 2, 4, 8, 16], # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size # and uses this to optimize the kernel. # FreeIndex[1] is usually letter "J" # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) - # Optimizations enabled by AssertFree1ElementMultiple>1: # - See above AssertFree0ElementMultiple "Load optimizations" - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree1ElementMultiple" : [1,2,4,8,16], - + "AssertFree1ElementMultiple": [1, 2, 4, 8, 16], # Assertions that require arithmetic intensity to be specified value. # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. "AssertAIGreaterThanEqual": -1, - "AssertAILessThanEqual": -1, - + "AssertAILessThanEqual": -1, # Stagger the start summation position of the tiles. # Elements from the summation dimension are loaded at offsets rather than all starting at 0. # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping @@ -729,7 +940,6 @@ # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. # In this case the start of each row of the tile is separated by an exact power-of-2 # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. - # StaggerU adjusts the start position in the summation (aka 'U') dimension # to avoid these conflicts. Both A and B matrix start at the adjusted position. # If >0 specifies the offset in multiples of the macro-tile "unroll" dim @@ -738,8 +948,7 @@ # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those # tiles start reading their summation dim parms. # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 - "StaggerU": [0,2,4,8,16,32,64], - + "StaggerU": [0, 2, 4, 8, 16, 32, 64], # Stride in bytes for each staggeru 'click'. # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so # each click will start in a new memory channel and spread traffic among the 16 available channels. @@ -749,8 +958,7 @@ # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. # (the implementation requires this - the unroll iteration accesses data in steps of # DepthU*BPE - "StaggerUStride": [-1,16,32,64,128,256,512,1024,2048], - + "StaggerUStride": [-1, 16, 32, 64, 128, 256, 512, 1024, 2048], # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: # 0: Use wg0 # 1: Use wg1 @@ -758,25 +966,21 @@ # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot # to a different bank since all workgroups still start at same point. - "StaggerUMapping": [0,1,2,3,4], - + "StaggerUMapping": [0, 1, 2, 3, 4], # GSU Workgroup Coalesced Ordering # False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} # True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} - "GlobalSplitUCoalesced": [False, True], - + "GlobalSplitUCoalesced": [False, True], # GSU Workgroup Mapping # False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} # -> workgroups do the summation by tile -> slower GR but faster GW # True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} # -> workgroups split up the summation -> faster GR but slower GW - "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], - + "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], # 0=don't use magic div (source only) # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 - "MagicDivAlg": [0,1,2], - + "MagicDivAlg": [0, 1, 2], # For Block Mapping type: # 0 : Use hardware-assigned wg number with no remapping. # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. @@ -801,22 +1005,31 @@ # # Formula for wgSerial: # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 - "WorkGroupMapping": list(range(-1024, 1024+1)), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best - "WorkGroupMappingXCC": [1,2,4,8,16,32], # change a workgroup's id so that contiguous workgroup can map on same XCC + "WorkGroupMapping": list( + range(-1024, 1024 + 1) + ), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best + "WorkGroupMappingXCC": [ + 1, + 2, + 4, + 8, + 16, + 32, + ], # change a workgroup's id so that contiguous workgroup can map on same XCC # -1 : WorkGroupMappingXCCGroup will be set to CU_count at runtime. Please ensure that (CU_count % WGMXCC == 0). - "WorkGroupMappingXCCGroup": list(range(-1, 1024)), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. - - "MaxOccupancy": list(range(1, 40+1)), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy - "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds - - #ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, + "WorkGroupMappingXCCGroup": list( + range(-1, 1024) + ), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. + "MaxOccupancy": list( + range(1, 40 + 1) + ), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy + "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds + # ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs - "ThreadTile": validThreadTiles, - "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 - - "WavefrontSize": [32, 64], - + "ThreadTile": validThreadTiles, + "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 + "WavefrontSize": [32, 64], # MatrixInstruction: (M x N x K x B) # XDLOPS tile definition, only valid for gfx908, gfx90a # MxNxKxB specifies matrix instruction variants @@ -839,8 +1052,7 @@ # MatrixInst BlkM WT Wave # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used - "MatrixInstruction": validMatrixInstructions, - + "MatrixInstruction": validMatrixInstructions, # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. # MI output data between each threads are along N dims. # But global memory is along M dim continuous. @@ -855,11 +1067,9 @@ # 1~8: Enable StoreRemap and set the global write vector width # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) # -1: Use dwordx2 if support SRVW, or set SRVW to 0 - "StoreRemapVectorWidth": [-1,0,1,2,4,8], - + "StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8], # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. - "SourceSwap": [False, True], - + "SourceSwap": [False, True], # Following parameters are designed for store scheduling. # (store stands for load from C (with beta) and store to C/D) # @@ -881,26 +1091,24 @@ # # WG1: _____________/?????\__________________ # |<------- loop -------->|<----- store ---->|end - "StorePriorityOpt": [False, True], + "StorePriorityOpt": [False, True], # # If we issue store in short period of time, kernel will become from compute bound to memory bound # 0 means issue instructions as many as possible if VGPR available - "NumElementsPerBatchStore": list(range(-1, 256)), + "NumElementsPerBatchStore": list(range(-1, 256)), # # add sync after per batch store in order to store contiguous elements # add sleep after per batch store in order to distribute store over whole loops # NOTE: this parameter is highly depends on size_k # 0 means no sync and sleep - "StoreSyncOpt": list(range(0, 256)), + "StoreSyncOpt": list(range(0, 256)), # # There are index or address calculation between global instructions. # issue global instruction b2b has better performance - "GroupLoadStore": [False, True], - + "GroupLoadStore": [False, True], # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. - "MIArchVgpr": [False, True], - + "MIArchVgpr": [False, True], # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel @@ -948,16 +1156,14 @@ # 2 = No partials # 3 = Nofixup and no partials "DebugStreamK": [0, 1, 2, 3], - # Controls desired width (#elements) for loads from global memory -> LDS. # and eliminates the pointer unshift logic # -1 : Set GlobalReadVectorWidth = VectorWidth # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) - "GlobalReadVectorWidthA": [ -2, -1, 1, 2, 3, 4, 6, 8, 16 ], - "GlobalReadVectorWidthB": [ -2, -1, 1, 2, 3, 4, 6, 8, 16 ], - + "GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16], + "GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16], # Controls desired width (#elements) for loads from LDS -> VGPR. # -1 : Set LocalReadVectorWidth = VectorWidth # 1 cannot be used for half type. @@ -967,9 +1173,7 @@ # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) - - "LocalReadVectorWidth": [ -1, 1, 2, 4, 8, 16 ], - + "LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16], # threads should read/write/operate on this many contiguous elements from the C matrix. # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) @@ -981,28 +1185,24 @@ # Using a VW too large which results in >16bytes/thread isn't supported # For MFMA non SourceSwap: this parameter didn't take effect # -1 means set vw to largest localReadWidth according to MIWaveTile - "VectorWidthA": [ -1, 1, 2, 3, 4, 6, 8 ], - "VectorWidthB": [ -1, 1, 2, 3, 4, 6, 8 ], - + "VectorWidthA": [-1, 1, 2, 3, 4, 6, 8], + "VectorWidthB": [-1, 1, 2, 3, 4, 6, 8], # If 0, store 1 element per instruction. # If 1, store vector-width elements per instruction. # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel - "VectorStore": [-1, 0, 1], - + "VectorStore": [-1, 0, 1], # Controls desired width (#elements) for stores from reg to global memory. # When MatrixInstruciton == None, derived parameter gwvw takes precedence. # -1 : Set StoreVectorWidth = VectorWidth - "StoreVectorWidth": [ -1, 1, 2, 3, 4, 6, 8 ], - + "StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8], # when loading all the data from global into lds requires multiple load instructions, these parameters govern which # loads will pull which rectangle of data from global into lds # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension - "NumLoadsCoalescedA": list(range(-1, 64+1)), - "NumLoadsCoalescedB": list(range(-1, 64+1)), - + "NumLoadsCoalescedA": list(range(-1, 64 + 1)), + "NumLoadsCoalescedB": list(range(-1, 64 + 1)), # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. @@ -1014,23 +1214,21 @@ # -1 : Only allow GLVW=1 # -2 : Only allow max(GLVWA,GLVWB) < VW ? # -3 : Only allow min(GLVWA,GLVWB) < VW ? - "DepthU": depthUs, - + "DepthU": depthUs, # integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting # performance so this has been deprecated and probably doesn't work # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 - "LdsPadA": [ -1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadB": [ -1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadMetadata": [ -1, 0, 1, 2, 3, 4, 8], + "LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8], # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). # Only support LdsBlockSizePerPad >= unrollDepth * BPE # 0 means disable LdsBlockSizePerPad - "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], - + "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], # Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) # -1 : keep LDS layout same as global fetch dimension for both A and B # set TLDS = 1 for NN,TN,TT @@ -1038,36 +1236,32 @@ # 0 : coalesced dimension of lds is tile dimension # 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected # 2 : coalesced dimension of lds is unroll dimension for both A and B - "TransposeLDS": [-1, 1, 0, 2], - + "TransposeLDS": [-1, 1, 0, 2], # add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit # 0: none, 1: glc, 2: slc, 3: glc slc # For gfx940, sets sc0/sc1 bits # 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1 - "NonTemporalE": list(range(0,8)), - "NonTemporalD": list(range(0,8)), - "NonTemporalC": list(range(0,8)), - "NonTemporalA": list(range(0,8)), - "NonTemporalB": list(range(0,8)), - "NonTemporalWS": list(range(0,8)), - "NonTemporalMetadata": list(range(0,8)), - "NonTemporal": list(range(-1,8)), - + "NonTemporalE": list(range(0, 8)), + "NonTemporalD": list(range(0, 8)), + "NonTemporalC": list(range(0, 8)), + "NonTemporalA": list(range(0, 8)), + "NonTemporalB": list(range(0, 8)), + "NonTemporalWS": list(range(0, 8)), + "NonTemporalMetadata": list(range(0, 8)), + "NonTemporal": list(range(-1, 8)), # Group together unroll iterations inside the unroll loop. # For example, InnerUnroll=2 will fetch LDS for two unroll iterations - "InnerUnroll": [1,2,4,8,16,32,64], - + "InnerUnroll": [1, 2, 4, 8, 16, 32, 64], # Enable CP preload kernel arguments feature # It can reduce time of loading kernel arguments by s_load. # It needs new complier and vbios to support this feature. - "PreloadKernArgs": [False, True], + "PreloadKernArgs": [False, True], # Kernels should be written in assembly or source # if assembly, ISA will determine architecture # if source, Runtime will determine language # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm - "KernelLanguage": [ "Assembly" ], - "ISA": validISA, # arch for assembly kernels - + "KernelLanguage": ["Assembly"], + "ISA": validISA, # arch for assembly kernels # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. @@ -1081,146 +1275,132 @@ # # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... - "CustomKernelName": -1, - + "CustomKernelName": -1, # Will allow a kernel to be accepted even when checks determine it's not viable. # Intended for use with custom kernels which have confirmed to be correct - "NoReject": [False, True], - + "NoReject": [False, True], # Debug use only. - "ActivationFused": [False, True], - + "ActivationFused": [False, True], # True- function call # False- inline - "ActivationFuncCall": [False, True], - + "ActivationFuncCall": [False, True], # Alternative implementation for activation function # Currently only supports GSU == 1 - "ActivationAlt": [False, True], - + "ActivationAlt": [False, True], # Do workgroup reduction. Currently for DBias - "WorkGroupReduction": [False], - + "WorkGroupReduction": [False], # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "Sparse": [0, 1, 2], - + "Sparse": [0, 1, 2], # in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds - "ConvertAfterDS": [False, True], - + "ConvertAfterDS": [False, True], # Force disable shadow init to release more sgpr in preloop - "ForceDisableShadowInit": [False, True], - } + "ForceDisableShadowInit": [False, True], +} # same parameter for all solution b/c depends only on compiler defaultBenchmarkCommonParameters = [ - {"InnerUnroll": [ 1 ] }, - {"KernelLanguage": [ "Assembly" ] }, - {"LdsPadA": [ -1 ] }, - {"LdsPadB": [ -1 ] }, - {"LdsPadMetadata": [ 0 ] }, - {"LdsBlockSizePerPadA": [ -1 ] }, - {"LdsBlockSizePerPadB": [ -1 ] }, - {"LdsBlockSizePerPadMetadata":[ 0 ] }, - {"TransposeLDS": [ -1 ] }, - {"MaxOccupancy": [ 40 ] }, - {"VectorWidthA": [ -1 ] }, - {"VectorWidthB": [ -1 ] }, - {"VectorStore": [ -1 ] }, - {"StoreVectorWidth": [ -1 ] }, - {"GlobalReadVectorWidthA": [ -1 ] }, - {"GlobalReadVectorWidthB": [ -1 ] }, - {"LocalReadVectorWidth": [ -1 ] }, - {"WaveSeparateGlobalReadA": [ 0 ] }, - {"WaveSeparateGlobalReadB": [ 0 ] }, - {"WaveSeparateGlobalReadMetadata": [ 0 ] }, - {"UnrollLoopSwapGlobalReadOrder": [ 0 ] }, - {"PrefetchGlobalRead": [ 1 ] }, - {"PrefetchLocalRead": [ 1 ] }, - {"ClusterLocalRead": [ 1 ] }, - {"SuppressNoLoadLoop": [ False ]}, - {"ExpandPointerSwap": [ True ]}, - - {"ScheduleGlobalRead": [ 1 ] }, - {"ScheduleLocalWrite": [ 1 ] }, - {"ScheduleIterAlg": [ 3 ] }, - - {"GlobalReadPerMfma": [ 1 ] }, - {"LocalWritePerMfma": [ -1 ] }, - - {"InterleaveAlpha": [ 0 ] }, - {"OptNoLoadLoop": [ 1 ] }, - - {"BufferLoad": [ True ] }, - {"BufferStore": [ True ] }, - {"DirectToVgprA": [ False ] }, - {"DirectToVgprB": [ False ] }, - {"DirectToVgprSparseMetadata":[ False ] }, - {"DirectToLds": [ False ] }, - {"UseSgprForGRO": [ -1 ] }, - {"UseInstOffsetForGRO": [ 0 ] }, - {"AssertSummationElementMultiple": [ 1 ] }, - {"AssertFree0ElementMultiple": [ 1 ] }, - {"AssertFree1ElementMultiple": [ 1 ] }, - - {"AssertAIGreaterThanEqual": [-1]}, - {"AssertAILessThanEqual": [-1]}, - - {"StaggerU": [ 32 ] }, # recommend [0,32] - {"StaggerUStride": [ 256 ] }, # recommend 256 for V10,V20 - {"StaggerUMapping": [ 0 ] }, # recommend [0,1] - {"MagicDivAlg": [ 2 ] }, - {"GlobalSplitU": [ 1 ] }, - {"GlobalSplitUAlgorithm": [ "MultipleBuffer" ] }, - {"GlobalSplitUCoalesced": [ False ] }, - {"GlobalSplitUWorkGroupMappingRoundRobin": [ False ] }, - {"Use64bShadowLimit": [ 1 ] }, - {"NumLoadsCoalescedA": [ 1 ] }, - {"NumLoadsCoalescedB": [ 1 ] }, - {"WorkGroup": [ [16,16,1]] }, - {"WorkGroupMapping": [ 8 ] }, - {"WorkGroupMappingXCC": [ 1 ] }, - {"WorkGroupMappingXCCGroup": [ -1 ] }, - {"ThreadTile": [ [4,4] ] }, - {"WavefrontSize": [ 64 ]}, - {"MatrixInstruction": [ [] ] }, - {"1LDSBuffer": [ 0 ] }, - {"DepthU": [ 16 ] }, - {"NonTemporalE": [ 0 ] }, - {"NonTemporalD": [ 0 ] }, - {"NonTemporalC": [ 0 ] }, - {"NonTemporalA": [ 0 ] }, - {"NonTemporalB": [ 0 ] }, - {"NonTemporalWS": [ 0 ] }, - {"NonTemporalMetadata": [ 0 ] }, - {"NonTemporal": [ -1 ] }, - {"PreloadKernArgs": [ True ] }, - {"CustomKernelName": [ "" ] }, - {"NoReject": [ False ]}, - {"StoreRemapVectorWidth": [ 0 ] }, - {"SourceSwap": [ False ] }, - {"StorePriorityOpt": [ False ] }, - {"NumElementsPerBatchStore": [ 0 ] }, - {"StoreSyncOpt": [ 0 ] }, - {"GroupLoadStore": [ False ] }, - {"MIArchVgpr": [ False ] }, - {"StreamK": [ 0 ] }, - {"StreamKAtomic": [ 0 ] }, - {"StreamKXCCMapping": [ 0 ] }, - {"DebugStreamK": [ 0 ] }, - {"ActivationFused": [ True ] }, - {"ActivationFuncCall": [ True ] }, - {"ActivationAlt": [ False ] }, - {"WorkGroupReduction": [ False ] }, - {"ConvertAfterDS": [ False ] }, - {"ForceDisableShadowInit": [ False ] } + {"InnerUnroll": [1]}, + {"KernelLanguage": ["Assembly"]}, + {"LdsPadA": [-1]}, + {"LdsPadB": [-1]}, + {"LdsPadMetadata": [0]}, + {"LdsBlockSizePerPadA": [-1]}, + {"LdsBlockSizePerPadB": [-1]}, + {"LdsBlockSizePerPadMetadata": [0]}, + {"TransposeLDS": [-1]}, + {"MaxOccupancy": [40]}, + {"VectorWidthA": [-1]}, + {"VectorWidthB": [-1]}, + {"VectorStore": [-1]}, + {"StoreVectorWidth": [-1]}, + {"GlobalReadVectorWidthA": [-1]}, + {"GlobalReadVectorWidthB": [-1]}, + {"LocalReadVectorWidth": [-1]}, + {"WaveSeparateGlobalReadA": [0]}, + {"WaveSeparateGlobalReadB": [0]}, + {"WaveSeparateGlobalReadMetadata": [0]}, + {"UnrollLoopSwapGlobalReadOrder": [0]}, + {"PrefetchGlobalRead": [1]}, + {"PrefetchLocalRead": [1]}, + {"ClusterLocalRead": [1]}, + {"SuppressNoLoadLoop": [False]}, + {"ExpandPointerSwap": [True]}, + {"ScheduleGlobalRead": [1]}, + {"ScheduleLocalWrite": [1]}, + {"ScheduleIterAlg": [3]}, + {"GlobalReadPerMfma": [1]}, + {"LocalWritePerMfma": [-1]}, + {"InterleaveAlpha": [0]}, + {"OptNoLoadLoop": [1]}, + {"BufferLoad": [True]}, + {"BufferStore": [True]}, + {"DirectToVgprA": [False]}, + {"DirectToVgprB": [False]}, + {"DirectToVgprSparseMetadata": [False]}, + {"DirectToLds": [False]}, + {"UseSgprForGRO": [-1]}, + {"UseInstOffsetForGRO": [0]}, + {"AssertSummationElementMultiple": [1]}, + {"AssertFree0ElementMultiple": [1]}, + {"AssertFree1ElementMultiple": [1]}, + {"AssertAIGreaterThanEqual": [-1]}, + {"AssertAILessThanEqual": [-1]}, + {"StaggerU": [32]}, # recommend [0,32] + {"StaggerUStride": [256]}, # recommend 256 for V10,V20 + {"StaggerUMapping": [0]}, # recommend [0,1] + {"MagicDivAlg": [2]}, + {"GlobalSplitU": [1]}, + {"GlobalSplitUAlgorithm": ["MultipleBuffer"]}, + {"GlobalSplitUCoalesced": [False]}, + {"GlobalSplitUWorkGroupMappingRoundRobin": [False]}, + {"Use64bShadowLimit": [1]}, + {"NumLoadsCoalescedA": [1]}, + {"NumLoadsCoalescedB": [1]}, + {"WorkGroup": [[16, 16, 1]]}, + {"WorkGroupMapping": [8]}, + {"WorkGroupMappingXCC": [1]}, + {"WorkGroupMappingXCCGroup": [-1]}, + {"ThreadTile": [[4, 4]]}, + {"WavefrontSize": [64]}, + {"MatrixInstruction": [[]]}, + {"1LDSBuffer": [0]}, + {"DepthU": [16]}, + {"NonTemporalE": [0]}, + {"NonTemporalD": [0]}, + {"NonTemporalC": [0]}, + {"NonTemporalA": [0]}, + {"NonTemporalB": [0]}, + {"NonTemporalWS": [0]}, + {"NonTemporalMetadata": [0]}, + {"NonTemporal": [-1]}, + {"PreloadKernArgs": [True]}, + {"CustomKernelName": [""]}, + {"NoReject": [False]}, + {"StoreRemapVectorWidth": [0]}, + {"SourceSwap": [False]}, + {"StorePriorityOpt": [False]}, + {"NumElementsPerBatchStore": [0]}, + {"StoreSyncOpt": [0]}, + {"GroupLoadStore": [False]}, + {"MIArchVgpr": [False]}, + {"StreamK": [0]}, + {"StreamKAtomic": [0]}, + {"StreamKXCCMapping": [0]}, + {"DebugStreamK": [0]}, + {"ActivationFused": [True]}, + {"ActivationFuncCall": [True]}, + {"ActivationAlt": [False]}, + {"WorkGroupReduction": [False]}, + {"ConvertAfterDS": [False]}, + {"ForceDisableShadowInit": [False]}, ] # dictionary of defaults comprised of default option for each parameter defaultSolution = {} for paramDict in defaultBenchmarkCommonParameters: - for key, value in paramDict.items(): - defaultSolution[key] = value[0] + for key, value in paramDict.items(): + defaultSolution[key] = value[0] # other non-benchmark options for solutions ################################################################################ @@ -1229,42 +1409,36 @@ defaultProblemType = { # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users # =TensorContraction requires specifying - "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights - - "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. - # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) - "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter - "UseE": False, # =True use output E to output gemm results before activation - "Gradient": False, # =True set globalWriteElements to gradient mode - "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction - "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. - "UseScaleAB": "", # Support "", "Scalar", and "Vector" - "UseScaleCD": False, # =True use scaleC, scaleD - "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction - "HighPrecisionAccumulate": False, # f32 += f16*f16 - "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. - - "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - - "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case - "ComplexConjugateB": False, - - "StochasticRounding": False, # By default, IEEE RNE rounding - + "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. + # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) + "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter + "UseE": False, # =True use output E to output gemm results before activation + "Gradient": False, # =True set globalWriteElements to gradient mode + "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction + "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. + "UseScaleAB": "", # Support "", "Scalar", and "Vector" + "UseScaleCD": False, # =True use scaleC, scaleD + "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction + "HighPrecisionAccumulate": False, # f32 += f16*f16 + "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. + "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case + "ComplexConjugateB": False, + "StochasticRounding": False, # By default, IEEE RNE rounding # for OperationType == GEMM - "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" - "TransposeB": True, - "Batched": False, # add batching dimension - "StridedBatched": True, # use to select general batch or strided batch - "GroupedGemm": False, # use to select general batch or strided batch - + "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" + "TransposeB": True, + "Batched": False, # add batching dimension + "StridedBatched": True, # use to select general batch or strided batch + "GroupedGemm": False, # use to select general batch or strided batch # for OperationType == TensorContraction # - Indices < NumIndicesC are Free or Batch indices and appear in C and D # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. @@ -1275,78 +1449,68 @@ # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. - "IndexAssignmentsA": [0, 2], - "IndexAssignmentsB": [1, 2], - "NumIndicesC": 2, - + "IndexAssignmentsA": [0, 2], + "IndexAssignmentsB": [1, 2], + "NumIndicesC": 2, # use initial strides for AB. # This has some performance impact for the increased flexibility: # - Additional strides will be passed into the kernel and will occupy SGPR registers # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) - "UseInitialStridesAB": False, - + "UseInitialStridesAB": False, # use initial strides for CD. # This has some performance impact for the increased flexibility: # - Additional strides will be passed into the kernel and will occupy SGPR registers # - Additional multiply on the store address path # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. - "UseInitialStridesCD": False, - - "AllowNoFreeDims": False, # allow A or B to specify no free dims - # (if false, A and B must have at least one free dim) - # (if true, A and B must have at least one free or batch dim) - + "UseInitialStridesCD": False, + "AllowNoFreeDims": False, # allow A or B to specify no free dims + # (if false, A and B must have at least one free dim) + # (if true, A and B must have at least one free or batch dim) # SetConstStride* sets the specified stride in the problem. # These no longer generate predicates - see AssertStrideEqualA/B below # List of pairs of [index, constValue]. # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets # strideA for index3 to constant '1' and stride for index2 to constant '4'. - "SetConstStrideA": [], - "SetConstStrideB": [], - "SetConstStrideBias": [], - + "SetConstStrideA": [], + "SetConstStrideB": [], + "SetConstStrideBias": [], # Summation dimension indices - "MirrorDimsA": [], - "MirrorDimsB": [], - "MirrorDimsMetadata": [], - + "MirrorDimsA": [], + "MirrorDimsB": [], + "MirrorDimsMetadata": [], # for LD description - "NumIndicesLD": 4, - "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB - + "NumIndicesLD": 4, + "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB # Tile aware solution selection - "TileAwareSelection": False, - + "TileAwareSelection": False, # Activation - "Activation": False, - "ActivationNoGuard": False, + "Activation": False, + "ActivationNoGuard": False, # AmaxD - "OutputAmaxD": False, + "OutputAmaxD": False, # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. - "SupportUserArgs": True, - "SwizzleTensorA": False, - "SwizzleTensorB": False, - } + "SupportUserArgs": True, + "SwizzleTensorA": False, + "SwizzleTensorB": False, +} -defaultProblemSizes = [{"Range": [ [2880], 0, 0 ]}] -defaultBenchmarkFinalProblemSizes = [{"Range": [ - [64, 64, 64, 512], 0, 0 ]}] -defaultBatchedProblemSizes = [{"Range": [ [2880], 0, [1], 0 ]}] -defaultBatchedBenchmarkFinalProblemSizes = [{"Range": [ - [64, 64, 64, 512], 0, [1], 0 ]}] +defaultProblemSizes = [{"Range": [[2880], 0, 0]}] +defaultBenchmarkFinalProblemSizes = [{"Range": [[64, 64, 64, 512], 0, 0]}] +defaultBatchedProblemSizes = [{"Range": [[2880], 0, [1], 0]}] +defaultBatchedBenchmarkFinalProblemSizes = [{"Range": [[64, 64, 64, 512], 0, [1], 0]}] -defaultSolutionSummationSizes = [32,64,96,128,256,512,1024,2048,4096,8192,16192] +defaultSolutionSummationSizes = [32, 64, 96, 128, 256, 512, 1024, 2048, 4096, 8192, 16192] ################################################################################ # Default Analysis Parameters ################################################################################ defaultAnalysisParameters = { - "ScheduleName": "Tensile", - "DeviceNames": "fallback", + "ScheduleName": "Tensile", + "DeviceNames": "fallback", "ArchitectureName": "gfx000", "LibraryType": "GridBased", - "SolutionImportanceMin": 0.01, # = 0.01=1% total time saved by keeping this solution - } + "SolutionImportanceMin": 0.01, # = 0.01=1% total time saved by keeping this solution +} diff --git a/tensilelite/Tensile/Common/Parallel.py b/tensilelite/Tensile/Common/Parallel.py index 6852746c55..9cd7396eb9 100644 --- a/tensilelite/Tensile/Common/Parallel.py +++ b/tensilelite/Tensile/Common/Parallel.py @@ -22,190 +22,224 @@ # ################################################################################ +import concurrent.futures import itertools import os import sys import time -import concurrent.futures from joblib import Parallel, delayed from .Utilities import tqdm + def joblibParallelSupportsGenerator(): - import joblib - from packaging.version import Version - joblibVer = joblib.__version__ - return Version(joblibVer) >= Version("1.4.0") + import joblib + from packaging.version import Version + + joblibVer = joblib.__version__ + return Version(joblibVer) >= Version("1.4.0") + def CPUThreadCount(enable=True): - from .GlobalParameters import globalParameters - if not enable: - return 1 - else: - if os.name == "nt": - cpu_count = os.cpu_count() + from .GlobalParameters import globalParameters + + if not enable: + return 1 else: - cpu_count = len(os.sched_getaffinity(0)) - cpuThreads = globalParameters["CpuThreads"] - if cpuThreads == -1: - return min(cpu_count, 64) # Temporarily hack to fix oom issue, remove this after jenkin is fixed. - return min(cpu_count, cpuThreads) + if os.name == "nt": + cpu_count = os.cpu_count() + else: + cpu_count = len(os.sched_getaffinity(0)) + cpuThreads = globalParameters["CpuThreads"] + if cpuThreads == -1: + return min( + cpu_count, 64 + ) # Temporarily hack to fix oom issue, remove this after jenkin is fixed. + return min(cpu_count, cpuThreads) + def pcallWithGlobalParamsMultiArg(f, args, newGlobalParameters): - OverwriteGlobalParameters(newGlobalParameters) - return f(*args) + OverwriteGlobalParameters(newGlobalParameters) + return f(*args) + def pcallWithGlobalParamsSingleArg(f, arg, newGlobalParameters): - OverwriteGlobalParameters(newGlobalParameters) - return f(arg) + OverwriteGlobalParameters(newGlobalParameters) + return f(arg) + def apply_print_exception(item, *args): - #print(item, args) - try: - if len(args) > 0: - func = item - args = args[0] - return func(*args) - else: - func, item = item - return func(item) - except Exception: - import traceback - traceback.print_exc() - raise - finally: - sys.stdout.flush() - sys.stderr.flush() + # print(item, args) + try: + if len(args) > 0: + func = item + args = args[0] + return func(*args) + else: + func, item = item + return func(item) + except Exception: + import traceback + + traceback.print_exc() + raise + finally: + sys.stdout.flush() + sys.stderr.flush() + def OverwriteGlobalParameters(newGlobalParameters): - from . import GlobalParameters - GlobalParameters.globalParameters.clear() - GlobalParameters.globalParameters.update(newGlobalParameters) + from . import GlobalParameters + + GlobalParameters.globalParameters.clear() + GlobalParameters.globalParameters.update(newGlobalParameters) + def ProcessingPool(enable=True, maxTasksPerChild=None): - import multiprocessing - import multiprocessing.dummy + import multiprocessing + import multiprocessing.dummy - threadCount = CPUThreadCount() + threadCount = CPUThreadCount() - if (not enable) or threadCount <= 1: - return multiprocessing.dummy.Pool(1) + if (not enable) or threadCount <= 1: + return multiprocessing.dummy.Pool(1) + + if multiprocessing.get_start_method() == "spawn": + from . import GlobalParameters + + return multiprocessing.Pool( + threadCount, + initializer=OverwriteGlobalParameters, + maxtasksperchild=maxTasksPerChild, + initargs=(GlobalParameters.globalParameters,), + ) + else: + return multiprocessing.Pool(threadCount, maxtasksperchild=maxTasksPerChild) - if multiprocessing.get_start_method() == "spawn": - from . import GlobalParameters - return multiprocessing.Pool(threadCount, initializer=OverwriteGlobalParameters, maxtasksperchild=maxTasksPerChild, initargs=(GlobalParameters.globalParameters,)) - else: - return multiprocessing.Pool(threadCount, maxtasksperchild=maxTasksPerChild) def ParallelMap(function, objects, message="", enable=True, method=None, maxTasksPerChild=None): - """ - Generally equivalent to list(map(function, objects)), possibly executing in parallel. - - message: A message describing the operation to be performed. - enable: May be set to false to disable parallelism. - method: A function which can fetch the mapping function from a processing pool object. - Leave blank to use .map(), other possiblities: - - `lambda x: x.starmap` - useful if `function` takes multiple parameters. - - `lambda x: x.imap` - lazy evaluation - - `lambda x: x.imap_unordered` - lazy evaluation, does not preserve order of return value. - """ - from .GlobalParameters import globalParameters - threadCount = CPUThreadCount(enable) - pool = ProcessingPool(enable, maxTasksPerChild) - - if threadCount <= 1 and globalParameters["ShowProgressBar"]: - # Provide a progress bar for single-threaded operation. - # This works for method=None, and for starmap. - mapFunc = map - if method is not None: - # itertools provides starmap which can fill in for pool.starmap. It provides imap on Python 2.7. - # If this works, we will use it, otherwise we will fallback to the "dummy" pool for single threaded - # operation. - try: - mapFunc = method(itertools) - except NameError: - mapFunc = None - - if mapFunc is not None: - return list(mapFunc(function, tqdm(objects, message))) - - mapFunc = pool.map - if method: mapFunc = method(pool) - - objects = zip(itertools.repeat(function), objects) - function = apply_print_exception - - countMessage = "" - try: - countMessage = " for {} tasks".format(len(objects)) - except TypeError: pass - - if message != "": message += ": " - - print("{0}Launching {1} threads{2}...".format(message, threadCount, countMessage)) - sys.stdout.flush() - currentTime = time.time() - rv = mapFunc(function, objects) - totalTime = time.time() - currentTime - print("{0}Done. ({1:.1f} secs elapsed)".format(message, totalTime)) - sys.stdout.flush() - pool.close() - return rv + """ + Generally equivalent to list(map(function, objects)), possibly executing in parallel. + + message: A message describing the operation to be performed. + enable: May be set to false to disable parallelism. + method: A function which can fetch the mapping function from a processing pool object. + Leave blank to use .map(), other possiblities: + - `lambda x: x.starmap` - useful if `function` takes multiple parameters. + - `lambda x: x.imap` - lazy evaluation + - `lambda x: x.imap_unordered` - lazy evaluation, does not preserve order of return value. + """ + from .GlobalParameters import globalParameters + + threadCount = CPUThreadCount(enable) + pool = ProcessingPool(enable, maxTasksPerChild) + + if threadCount <= 1 and globalParameters["ShowProgressBar"]: + # Provide a progress bar for single-threaded operation. + # This works for method=None, and for starmap. + mapFunc = map + if method is not None: + # itertools provides starmap which can fill in for pool.starmap. It provides imap on Python 2.7. + # If this works, we will use it, otherwise we will fallback to the "dummy" pool for single threaded + # operation. + try: + mapFunc = method(itertools) + except NameError: + mapFunc = None + + if mapFunc is not None: + return list(mapFunc(function, tqdm(objects, message))) + + mapFunc = pool.map + if method: + mapFunc = method(pool) + + objects = zip(itertools.repeat(function), objects) + function = apply_print_exception + + countMessage = "" + try: + countMessage = " for {} tasks".format(len(objects)) + except TypeError: + pass + + if message != "": + message += ": " + + print("{0}Launching {1} threads{2}...".format(message, threadCount, countMessage)) + sys.stdout.flush() + currentTime = time.time() + rv = mapFunc(function, objects) + totalTime = time.time() - currentTime + print("{0}Done. ({1:.1f} secs elapsed)".format(message, totalTime)) + sys.stdout.flush() + pool.close() + return rv + def ParallelMapReturnAsGenerator(function, objects, message="", enable=True, multiArg=True): - from .GlobalParameters import globalParameters - threadCount = CPUThreadCount(enable) - print("{0}Launching {1} threads...".format(message, threadCount)) + from .GlobalParameters import globalParameters + + threadCount = CPUThreadCount(enable) + print("{0}Launching {1} threads...".format(message, threadCount)) - if threadCount <= 1 and globalParameters["ShowProgressBar"]: - # Provide a progress bar for single-threaded operation. - callFunc = lambda args: function(*args) if multiArg else lambda args: function(args) - return [callFunc(args) for args in tqdm(objects, message)] + if threadCount <= 1 and globalParameters["ShowProgressBar"]: + # Provide a progress bar for single-threaded operation. + callFunc = lambda args: function(*args) if multiArg else lambda args: function(args) + return [callFunc(args) for args in tqdm(objects, message)] + + with concurrent.futures.ProcessPoolExecutor(max_workers=threadCount) as executor: + resultFutures = (executor.submit(function, *arg if multiArg else arg) for arg in objects) + for result in concurrent.futures.as_completed(resultFutures): + yield result.result() - with concurrent.futures.ProcessPoolExecutor(max_workers=threadCount) as executor: - resultFutures = (executor.submit(function, *arg if multiArg else arg) for arg in objects) - for result in concurrent.futures.as_completed(resultFutures): - yield result.result() def ParallelMap2(function, objects, message="", enable=True, multiArg=True, return_as="list"): - """ - Generally equivalent to list(map(function, objects)), possibly executing in parallel. - - message: A message describing the operation to be performed. - enable: May be set to false to disable parallelism. - multiArg: True if objects represent multiple arguments - (differentiates multi args vs single collection arg) - """ - if return_as in ('generator', 'generator_unordered') and not joblibParallelSupportsGenerator(): - return ParallelMapReturnAsGenerator(function, objects, message, enable, multiArg) - - from .GlobalParameters import globalParameters - threadCount = CPUThreadCount(enable) - - if threadCount <= 1 and globalParameters["ShowProgressBar"]: - # Provide a progress bar for single-threaded operation. - return [function(*args) if multiArg else function(args) for args in tqdm(objects, message)] - - countMessage = "" - try: - countMessage = " for {} tasks".format(len(objects)) - except TypeError: pass - - if message != "": message += ": " - print("{0}Launching {1} threads{2}...".format(message, threadCount, countMessage)) - sys.stdout.flush() - currentTime = time.time() - - pcall = pcallWithGlobalParamsMultiArg if multiArg else pcallWithGlobalParamsSingleArg - pargs = zip(objects, itertools.repeat(globalParameters)) - - if joblibParallelSupportsGenerator(): - rv = Parallel(n_jobs=threadCount,timeout=99999, return_as=return_as)(delayed(pcall)(function, a, params) for a, params in pargs) - else: - rv = Parallel(n_jobs=threadCount,timeout=99999)(delayed(pcall)(function, a, params) for a, params in pargs) - - totalTime = time.time() - currentTime - print("{0}Done. ({1:.1f} secs elapsed)".format(message, totalTime)) - sys.stdout.flush() - return rv + """ + Generally equivalent to list(map(function, objects)), possibly executing in parallel. + + message: A message describing the operation to be performed. + enable: May be set to false to disable parallelism. + multiArg: True if objects represent multiple arguments + (differentiates multi args vs single collection arg) + """ + if return_as in ("generator", "generator_unordered") and not joblibParallelSupportsGenerator(): + return ParallelMapReturnAsGenerator(function, objects, message, enable, multiArg) + + from .GlobalParameters import globalParameters + + threadCount = CPUThreadCount(enable) + + if threadCount <= 1 and globalParameters["ShowProgressBar"]: + # Provide a progress bar for single-threaded operation. + return [function(*args) if multiArg else function(args) for args in tqdm(objects, message)] + + countMessage = "" + try: + countMessage = " for {} tasks".format(len(objects)) + except TypeError: + pass + + if message != "": + message += ": " + print("{0}Launching {1} threads{2}...".format(message, threadCount, countMessage)) + sys.stdout.flush() + currentTime = time.time() + + pcall = pcallWithGlobalParamsMultiArg if multiArg else pcallWithGlobalParamsSingleArg + pargs = zip(objects, itertools.repeat(globalParameters)) + + if joblibParallelSupportsGenerator(): + rv = Parallel(n_jobs=threadCount, timeout=99999, return_as=return_as)( + delayed(pcall)(function, a, params) for a, params in pargs + ) + else: + rv = Parallel(n_jobs=threadCount, timeout=99999)( + delayed(pcall)(function, a, params) for a, params in pargs + ) + + totalTime = time.time() - currentTime + print("{0}Done. ({1:.1f} secs elapsed)".format(message, totalTime)) + sys.stdout.flush() + return rv diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index 8bb6eec02b..ddb6900e3d 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -1,7 +1,8 @@ -from typing import Tuple, NamedTuple +from typing import NamedTuple, Tuple IsaVersion = Tuple[int, int, int] + class SemanticVersion(NamedTuple): major: int minor: int diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 4ed27a4c9e..3b746a6f94 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -1,141 +1,161 @@ +import functools +import math import os import re +import subprocess import sys -import math import time -import functools import warnings -import subprocess - from copy import deepcopy from enum import Enum from typing import Optional -from .Capabilities import initAsmBugs, initArchCaps, initAsmCaps -from .Architectures import getGfxArch, getGfxName -from .GlobalParameters import globalParameters, defaultGlobalParameters, validParameters, __version__ +from .Architectures import gfxToIsa, isaToGfx +from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps +from .GlobalParameters import ( + __version__, + defaultGlobalParameters, + globalParameters, + validParameters, +) + # get param values from structures. -def hasParam( name, structure ): - if isinstance(structure, list): - for l in structure: - if hasParam(name, l): - return True - return False - elif isinstance(structure, dict): - return name in structure - else: - return name == structure - #printExit("structure %s is not list or dict" % structure) +def hasParam(name, structure): + if isinstance(structure, list): + for l in structure: + if hasParam(name, l): + return True + return False + elif isinstance(structure, dict): + return name in structure + else: + return name == structure + # printExit("structure %s is not list or dict" % structure) + ################################################################################ # Print Debug ################################################################################ def print1(message): - if globalParameters["PrintLevel"] >= 1: - print(message) - sys.stdout.flush() + if globalParameters["PrintLevel"] >= 1: + print(message) + sys.stdout.flush() + + def print2(message): - if globalParameters["PrintLevel"] >= 2: - print(message) - sys.stdout.flush() + if globalParameters["PrintLevel"] >= 2: + print(message) + sys.stdout.flush() + def printWarning(message): - print("Tensile::WARNING: %s" % message) - sys.stdout.flush() + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() + + def printExit(message): - print("Tensile::FATAL: %s" % message) - sys.stdout.flush() - sys.exit(-1) + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) + ################################################################################ # Locate Executables # rocm-smi, hip-clang, rocm_agent_enumerator, clang-offload-bundler ################################################################################ -def isExe( filePath ): - return os.path.isfile(filePath) and os.access(filePath, os.X_OK) -def locateExe( defaultPath, exeName ): # /opt/rocm/bin, hip-clang - # look in defaultPath first - exePath = os.path.join(defaultPath, exeName) - if isExe(exePath): - return exePath - # look in PATH second - for path in os.environ["PATH"].split(os.pathsep): - exePath = os.path.join(path, exeName) +def isExe(filePath): + return os.path.isfile(filePath) and os.access(filePath, os.X_OK) + + +def locateExe(defaultPath, exeName): # /opt/rocm/bin, hip-clang + # look in defaultPath first + exePath = os.path.join(defaultPath, exeName) if isExe(exePath): - return exePath - return None + return exePath + # look in PATH second + for path in os.environ["PATH"].split(os.pathsep): + exePath = os.path.join(path, exeName) + if isExe(exePath): + return exePath + return None def which(p): - if 'CMAKE_CXX_COMPILER' in os.environ and os.path.isfile(os.environ['CMAKE_CXX_COMPILER']): - return os.environ['CMAKE_CXX_COMPILER'] + if "CMAKE_CXX_COMPILER" in os.environ and os.path.isfile(os.environ["CMAKE_CXX_COMPILER"]): + return os.environ["CMAKE_CXX_COMPILER"] if os.name == "nt": - exes = [p+x for x in ['.exe', '', '.bat']] # bat may be front end for file with no extension + exes = [ + p + x for x in [".exe", "", ".bat"] + ] # bat may be front end for file with no extension else: - exes = [p+x for x in ['', '.exe', '.bat']] - system_path = os.environ['PATH'].split(os.pathsep) - for dirname in system_path+[globalParameters["ROCmBinPath"]]: + exes = [p + x for x in ["", ".exe", ".bat"]] + system_path = os.environ["PATH"].split(os.pathsep) + for dirname in system_path + [globalParameters["ROCmBinPath"]]: for exe in exes: candidate = os.path.join(os.path.expanduser(dirname), exe) if os.path.isfile(candidate): return candidate return None + def splitArchs(fromTensile=False): - # Helper for architecture - def isSupported(arch): - return globalParameters["AsmCaps"][arch]["SupportedISA"] and \ - globalParameters["AsmCaps"][arch]["SupportedSource"] - - if ";" in globalParameters["Architecture"]: - wantedArchs = globalParameters["Architecture"].split(";") - else: - wantedArchs = globalParameters["Architecture"].split("_") - archs = [] - cmdlineArchs = [] - if "all" in wantedArchs: - for arch in globalParameters['SupportedISA']: - if isSupported(arch): - if (arch in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]): - if (arch == (9,0,10)): - archs += [getGfxName(arch) + '-xnack+'] - cmdlineArchs += [getGfxName(arch) + ':xnack+'] - if globalParameters["AsanBuild"]: - archs += [getGfxName(arch) + '-xnack+'] - cmdlineArchs += [getGfxName(arch) + ':xnack+'] - else: - archs += [getGfxName(arch) + '-xnack-'] - cmdlineArchs += [getGfxName(arch) + ':xnack-'] - else: - archs += [getGfxName(arch)] - cmdlineArchs += [getGfxName(arch)] - else: - for arch in wantedArchs: - archs += [re.sub(":", "-", arch)] - cmdlineArchs += [arch] + # Helper for architecture + def isSupported(arch): + return ( + globalParameters["AsmCaps"][arch]["SupportedISA"] + and globalParameters["AsmCaps"][arch]["SupportedSource"] + ) + + if ";" in globalParameters["Architecture"]: + wantedArchs = globalParameters["Architecture"].split(";") + else: + wantedArchs = globalParameters["Architecture"].split("_") + archs = [] + cmdlineArchs = [] + if "all" in wantedArchs: + for arch in globalParameters["SupportedISA"]: + if isSupported(arch): + if arch in [(9, 0, 6), (9, 0, 8), (9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)]: + if arch == (9, 0, 10): + archs += [isaToGfx(arch) + "-xnack+"] + cmdlineArchs += [isaToGfx(arch) + ":xnack+"] + if globalParameters["AsanBuild"]: + archs += [isaToGfx(arch) + "-xnack+"] + cmdlineArchs += [isaToGfx(arch) + ":xnack+"] + else: + archs += [isaToGfx(arch) + "-xnack-"] + cmdlineArchs += [isaToGfx(arch) + ":xnack-"] + else: + archs += [isaToGfx(arch)] + cmdlineArchs += [isaToGfx(arch)] + else: + for arch in wantedArchs: + archs += [re.sub(":", "-", arch)] + cmdlineArchs += [arch] - # if calling from the context of Tensile we only want the arch associated with the current ISA - if fromTensile: - gfx = getGfxName(globalParameters["CurrentISA"]) - archs = set(a for a in archs if gfx in a) - cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) + # if calling from the context of Tensile we only want the arch associated with the current ISA + if fromTensile: + gfx = isaToGfx(globalParameters["CurrentISA"]) + archs = set(a for a in archs if gfx in a) + cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) - return archs, cmdlineArchs + return archs, cmdlineArchs def ensurePath(path): - try: - os.makedirs(path) - except FileExistsError: - pass - except OSError: - printExit("Failed to create directory \"%s\" " % (path) ) - return path + try: + os.makedirs(path) + except FileExistsError: + pass + except OSError: + printExit('Failed to create directory "%s" ' % (path)) + return path def roundUp(f): - return (int)(math.ceil(f)) + return (int)(math.ceil(f)) + ################################################################################ # Is query version compatible with current version @@ -143,20 +163,20 @@ def roundUp(f): # tensile.major == yaml.major and tensile.minor.step > yaml.minor.step ################################################################################ def versionIsCompatible(queryVersionString): - (qMajor, qMinor, qStep) = queryVersionString.split(".") - (tMajor, tMinor, tStep) = __version__.split(".") + (qMajor, qMinor, qStep) = queryVersionString.split(".") + (tMajor, tMinor, tStep) = __version__.split(".") - # major version must match exactly - if qMajor != tMajor: - return False + # major version must match exactly + if qMajor != tMajor: + return False - # minor.patch version must be >= - if int(qMinor) > int(tMinor): - return False - if qMinor == tMinor: - if int(qStep) > int(tStep): - return False - return True + # minor.patch version must be >= + if int(qMinor) > int(tMinor): + return False + if qMinor == tMinor: + if int(qStep) > int(tStep): + return False + return True def checkParametersAreValid(param, validParams): @@ -168,76 +188,86 @@ def checkParametersAreValid(param, validParams): return if name not in validParams: - printExit("Invalid parameter name: {}\nValid parameters are {}." \ - .format(name, sorted(validParameters.keys()))) + printExit( + "Invalid parameter name: {}\nValid parameters are {}.".format( + name, sorted(validParameters.keys()) + ) + ) for value in values: if validParams[name] != -1 and value not in validParams[name]: msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ - if len(validParams[name])>32 else "" + msgExt = ( + " (only first 32 combos printed)\nRefer to Common.py for more info" + if len(validParams[name]) > 32 + else "" + ) printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) + ################################################################################ # Progress Bar Printing # prints "||||" up to width ################################################################################ class ProgressBar: - def __init__(self, maxValue, width=80): - self.char = '|' - self.maxValue = maxValue - self.width = width - self.maxTicks = self.width - 7 - - - self.priorValue = 0 - self.fraction = 0 - self.numTicks = 0 - self.createTime = time.time() - - def increment(self, value=1): - self.update(self.priorValue+value) - - def update(self, value): - currentFraction = 1.0 * value / self.maxValue - currentNumTicks = int(currentFraction * self.maxTicks) - if currentNumTicks > self.numTicks: - self.numTicks = currentNumTicks - self.fraction = currentFraction - self.printStatus() - self.priorValue = value - - def printStatus(self): - sys.stdout.write("\r") - sys.stdout.write("[%-*s] %3d%%" \ - % (self.maxTicks, self.char*self.numTicks, self.fraction*100) ) - if self.numTicks == self.maxTicks: - stopTime = time.time() - sys.stdout.write(" (%-.1f secs elapsed)\n"%(stopTime-self.createTime)) - sys.stdout.flush() + def __init__(self, maxValue, width=80): + self.char = "|" + self.maxValue = maxValue + self.width = width + self.maxTicks = self.width - 7 - def finish(self): pass + self.priorValue = 0 + self.fraction = 0 + self.numTicks = 0 + self.createTime = time.time() + + def increment(self, value=1): + self.update(self.priorValue + value) + + def update(self, value): + currentFraction = 1.0 * value / self.maxValue + currentNumTicks = int(currentFraction * self.maxTicks) + if currentNumTicks > self.numTicks: + self.numTicks = currentNumTicks + self.fraction = currentFraction + self.printStatus() + self.priorValue = value + + def printStatus(self): + sys.stdout.write("\r") + sys.stdout.write( + "[%-*s] %3d%%" % (self.maxTicks, self.char * self.numTicks, self.fraction * 100) + ) + if self.numTicks == self.maxTicks: + stopTime = time.time() + sys.stdout.write(" (%-.1f secs elapsed)\n" % (stopTime - self.createTime)) + sys.stdout.flush() + + def finish(self): + pass class DataDirection(Enum): - NONE = 0, - READ = 1, - WRITE = 2 + NONE = (0,) + READ = (1,) + WRITE = 2 + class SpinnyThing: def __init__(self): - self.chars = ['|', '/', '-', '\\'] + self.chars = ["|", "/", "-", "\\"] self.index = 0 def increment(self, value=1): - sys.stdout.write('\b' + self.chars[self.index]) + sys.stdout.write("\b" + self.chars[self.index]) sys.stdout.flush() self.index = (self.index + 1) % len(self.chars) def finish(self): - sys.stdout.write('\b*\n') + sys.stdout.write("\b*\n") sys.stdout.flush() + def iterate_progress(obj, *args, **kwargs): try: progress = ProgressBar(len(obj)) @@ -248,16 +278,18 @@ def iterate_progress(obj, *args, **kwargs): progress.increment() progress.finish() + try: from tqdm import tqdm except ImportError: tqdm = iterate_progress + def state(obj): - if hasattr(obj, 'state'): + if hasattr(obj, "state"): return obj.state() - if hasattr(obj.__class__, 'StateKeys'): + if hasattr(obj.__class__, "StateKeys"): rv = {} for key in obj.__class__.StateKeys: attr = key @@ -279,12 +311,14 @@ def state(obj): return obj + def state_key_ordering(cls): def tup(obj): return tuple([getattr(obj, k) for k in cls.StateKeys]) def lt(a, b): return tup(a) < tup(b) + def eq(a, b): return tup(a) == tup(b) @@ -293,10 +327,11 @@ def eq(a, b): return functools.total_ordering(cls) + def hash_combine(*objs, **kwargs): shift = 1 - if 'shift' in kwargs: - shift = kwargs['shift'] + if "shift" in kwargs: + shift = kwargs["shift"] if len(objs) == 1: objs = objs[0] @@ -313,12 +348,16 @@ def hash_combine(*objs, **kwargs): pass return rv + def hash_objs(*objs, **kwargs): return hash(tuple(objs)) + def showwarning(message, category, filename, lineno, file=None, line=None): msg = f"> {category.__name__}: {message}" print(msg) + + warnings.showwarning = showwarning @@ -328,259 +367,297 @@ def showwarning(message, category, filename, lineno, file=None, line=None): # tensile.major == yaml.major and tensile.minor.step > yaml.minor.step ################################################################################ def detectGlobalCurrentISA_(detectionTool): - """ - Returns returncode if detection failure - """ - global globalParameters - - if globalParameters["CurrentISA"] == (0,0,0) and detectionTool: - process = subprocess.run([detectionTool], stdout=subprocess.PIPE) - if os.name == "nt": - line = "" - for line_in in process.stdout.decode().splitlines(): - if 'gcnArchName' in line_in: - line += line_in.split()[1] - break # detemine if hipinfo will support multiple arch - arch = getGfxArch(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + getGfxName(arch)) - globalParameters["CurrentISA"] = arch - else: - archList = [] - for line in process.stdout.decode().split("\n"): - arch = getGfxArch(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + getGfxName(arch)) - archList.append(arch) - if len(archList) > 0: - globalParameters["CurrentISA"] = archList[globalParameters["Device"]] - if (process.returncode): - printWarning("%s exited with code %u" % (detectionTool, process.returncode)) - return process.returncode - return 0 + """ + Returns returncode if detection failure + """ + global globalParameters + + if globalParameters["CurrentISA"] == (0, 0, 0) and detectionTool: + process = subprocess.run([detectionTool], stdout=subprocess.PIPE) + if os.name == "nt": + line = "" + for line_in in process.stdout.decode().splitlines(): + if "gcnArchName" in line_in: + line += line_in.split()[1] + break # detemine if hipinfo will support multiple arch + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + isaToGfx(arch)) + globalParameters["CurrentISA"] = arch + else: + archList = [] + for line in process.stdout.decode().split("\n"): + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + isaToGfx(arch)) + archList.append(arch) + if len(archList) > 0: + globalParameters["CurrentISA"] = archList[globalParameters["Device"]] + if process.returncode: + printWarning("%s exited with code %u" % (detectionTool, process.returncode)) + return process.returncode + return 0 def detectGlobalCurrentISA(): - """ - Returns returncode if detection failure - """ - errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) - if errorCode: - printWarning("Attempting to detect ISA with rocm_agent_enumerator") - return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) - return errorCode + """ + Returns returncode if detection failure + """ + errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) + if errorCode: + printWarning("Attempting to detect ISA with rocm_agent_enumerator") + return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) + return errorCode def restoreDefaultGlobalParameters(): - """ - Restores `globalParameters` back to defaults. - """ - global globalParameters - global defaultGlobalParameters - # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would - # result in dangling references, specifically in Tensile.Tensile(). - globalParameters.clear() - for key, value in deepcopy(defaultGlobalParameters).items(): - globalParameters[key] = value + """ + Restores `globalParameters` back to defaults. + """ + global globalParameters + global defaultGlobalParameters + # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would + # result in dangling references, specifically in Tensile.Tensile(). + globalParameters.clear() + for key, value in deepcopy(defaultGlobalParameters).items(): + globalParameters[key] = value + def printTable(rows): - rows = list([[str(cell) for cell in row] for row in rows]) - colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) + rows = list([[str(cell) for cell in row] for row in rows]) + colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - for row in rows: - for (width, cell) in zip(colWidths, row): - pad = ' ' * (width - len(cell)) - print(pad, cell, sep='', end=' ') - print() + for row in rows: + for width, cell in zip(colWidths, row): + pad = " " * (width - len(cell)) + print(pad, cell, sep="", end=" ") + print() def printCapTable(parameters): - import itertools - archs = [(0,0,0)] + parameters["SupportedISA"] - gfxNames = list(map(getGfxName, archs)) + import itertools + + archs = [(0, 0, 0)] + parameters["SupportedISA"] + gfxNames = list(map(isaToGfx, archs)) - headerRow = ['cap'] + gfxNames + headerRow = ["cap"] + gfxNames - def capRow(caps, cap): - return [cap] + [('1' if cap in caps[arch] and caps[arch][cap] else '0') for arch in archs] + def capRow(caps, cap): + return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] - allAsmCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()])) - allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) - asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + allAsmCaps = set( + itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) + ) + allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) + asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] - allArchCaps = set(itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()])) - allArchCaps = sorted(allArchCaps) - archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + allArchCaps = set( + itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) + ) + allArchCaps = sorted(allArchCaps) + archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] - printTable([headerRow] + asmCapRows + archCapRows) + printTable([headerRow] + asmCapRows + archCapRows) def assignGlobalParameters(config, cxxCompiler=None): - """ - Assign Global Parameters - Each global parameter has a default parameter, and the user - can override them, those overridings happen here - """ - - global globalParameters - - # Minimum Required Version - if "MinimumRequiredVersion" in config: - if not versionIsCompatible(config["MinimumRequiredVersion"]): - printExit("Config file requires version=%s is not compatible with current Tensile version=%s" \ - % (config["MinimumRequiredVersion"], __version__) ) - - # User-specified global parameters - print2("GlobalParameters:") - for key in globalParameters: - defaultValue = globalParameters[key] - if key in config: - configValue = config[key] - if configValue == defaultValue: - print2(" %24s: %8s (same)" % (key, configValue)) - else: - print2(" %24s: %8s (overriden)" % (key, configValue)) - else: - print2(" %24s: %8s (unspecified)" % (key, defaultValue)) - - globalParameters["ROCmPath"] = "/opt/rocm" - if "ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") - if "TENSILE_ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") - if os.name == "nt" and "HIP_DIR" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM - globalParameters["CmakeCxxCompiler"] = None - if "CMAKE_CXX_COMPILER" in os.environ: - globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") - if "CMAKE_C_COMPILER" in os.environ: - globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") - - globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") - - # ROCm AMD GPU Arch Path - # ROCm Agent Enumerator Path - if os.name == "nt": - globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") - globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "hipinfo.exe") - else: - globalParameters["AMDGPUArchPath"] = locateExe(globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch") - globalParameters["ROCmAgentEnumeratorPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm_agent_enumerator") - - globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") - globalParameters["ROCmLdPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld") - - globalParameters["ExtractKernelPath"] = locateExe(os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel") - - if "AMDGPUArchPath" in config: - globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] - - if "AsanBuild" in config: - globalParameters["AsanBuild"] = config["AsanBuild"] - - if "KeepBuildTmp" in config: - globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] - - if "CodeObjectVersion" in config: - globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - - # read current gfx version - returncode = detectGlobalCurrentISA() - if globalParameters["CurrentISA"] == (0,0,0): - printWarning("Did not detect SupportedISA: %s; cannot benchmark assembly kernels." % globalParameters["SupportedISA"]) - if returncode: + """ + Assign Global Parameters + Each global parameter has a default parameter, and the user + can override them, those overridings happen here + """ + + global globalParameters + + # Minimum Required Version + if "MinimumRequiredVersion" in config: + if not versionIsCompatible(config["MinimumRequiredVersion"]): + printExit( + "Config file requires version=%s is not compatible with current Tensile version=%s" + % (config["MinimumRequiredVersion"], __version__) + ) + + # User-specified global parameters + print2("GlobalParameters:") + for key in globalParameters: + defaultValue = globalParameters[key] + if key in config: + configValue = config[key] + if configValue == defaultValue: + print2(" %24s: %8s (same)" % (key, configValue)) + else: + print2(" %24s: %8s (overriden)" % (key, configValue)) + else: + print2(" %24s: %8s (unspecified)" % (key, defaultValue)) + + globalParameters["ROCmPath"] = "/opt/rocm" + if "ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") + if "TENSILE_ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") + if os.name == "nt" and "HIP_DIR" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM + globalParameters["CmakeCxxCompiler"] = None + if "CMAKE_CXX_COMPILER" in os.environ: + globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") + if "CMAKE_C_COMPILER" in os.environ: + globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") + + globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") + + # ROCm AMD GPU Arch Path + # ROCm Agent Enumerator Path if os.name == "nt": - globalParameters["CurrentISA"] = (9,0,6) - printWarning("Failed to detect ISA so forcing (gfx906) on windows") - - globalParameters["AsmCaps"] = {} - globalParameters["ArchCaps"] = {} - globalParameters["AsmBugs"] = {} - - for v in globalParameters["SupportedISA"] + [(0,0,0)]: - globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) - globalParameters["ArchCaps"][v] = initArchCaps(v) - globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) - - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) + globalParameters["AMDGPUArchPath"] = locateExe( + globalParameters["ROCmBinPath"], "hipinfo.exe" + ) + globalParameters["ROCmAgentEnumeratorPath"] = locateExe( + globalParameters["ROCmBinPath"], "hipinfo.exe" + ) + else: + globalParameters["AMDGPUArchPath"] = locateExe( + globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch" + ) + globalParameters["ROCmAgentEnumeratorPath"] = locateExe( + globalParameters["ROCmBinPath"], "rocm_agent_enumerator" + ) + + globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") + globalParameters["ROCmLdPath"] = locateExe( + os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld" + ) + + globalParameters["ExtractKernelPath"] = locateExe( + os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel" + ) + + if "AMDGPUArchPath" in config: + globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] + + if "AsanBuild" in config: + globalParameters["AsanBuild"] = config["AsanBuild"] + + if "KeepBuildTmp" in config: + globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] + + if "CodeObjectVersion" in config: + globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] + + # read current gfx version + returncode = detectGlobalCurrentISA() + if globalParameters["CurrentISA"] == (0, 0, 0): + printWarning( + "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." + % globalParameters["SupportedISA"] + ) + if returncode: + if os.name == "nt": + globalParameters["CurrentISA"] = (9, 0, 6) + printWarning("Failed to detect ISA so forcing (gfx906) on windows") + + globalParameters["AsmCaps"] = {} + globalParameters["ArchCaps"] = {} + globalParameters["AsmBugs"] = {} + + for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: + globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) + globalParameters["ArchCaps"][v] = initArchCaps(v) + globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + + if globalParameters["PrintLevel"] >= 1: + printCapTable(globalParameters) + + globalParameters["SupportedISA"] = list( + [ + i + for i in globalParameters["SupportedISA"] + if globalParameters["AsmCaps"][i]["SupportedISA"] + ] + ) + + validParameters["ISA"] = [(0, 0, 0), *globalParameters["SupportedISA"]] + + # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future + # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of + # '0.0.0' will persist + + # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. + # The alternative would be to install the `distro` package. + # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution + + # The following try except block computes the hipcc version + try: + if os.name == "nt": + compileArgs = ["perl"] + [which("hipcc")] + ["--version"] + output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() + else: + compiler = "hipcc" + output = subprocess.run( + [compiler, "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode() + + for line in output.split("\n"): + if "HIP version" in line: + globalParameters["HipClangVersion"] = line.split()[2] + print1("# Found hipcc version " + globalParameters["HipClangVersion"]) + + except (subprocess.CalledProcessError, OSError) as e: + printWarning("Error: {} running {} {} ".format("hipcc", "--version", e)) + + # The following keys may be present in the config, but are not (or no longer) global parameters. + ignoreKeys = [ + "UseCompression", + "CxxCompiler", + "CCompiler", + "OffloadBundler", + "Assembler", + "LogicPath", + "LogicFilter", + "OutputPath", + "Experimental", + "GenSolTable", + ] + for key in config: + if key in ignoreKeys: + continue + value = config[key] + if key not in globalParameters: + printWarning("Global parameter %s = %s unrecognised." % (key, value)) + globalParameters[key] = value - globalParameters["SupportedISA"] = list([i for i in globalParameters["SupportedISA"] if globalParameters["AsmCaps"][i]["SupportedISA"]]) - validParameters["ISA"] = [(0,0,0), *globalParameters["SupportedISA"]] +def setupRestoreClocks(): + import atexit - # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future - # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of - # '0.0.0' will persist + def restoreClocks(): + if globalParameters["PinClocks"]: + rsmi = globalParameters["ROCmSMIPath"] + subprocess.call([rsmi, "-d", "0", "--resetclocks"]) + subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) - # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. - # The alternative would be to install the `distro` package. - # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution + atexit.register(restoreClocks) - # The following try except block computes the hipcc version - try: - if os.name == "nt": - compileArgs = ['perl'] + [which('hipcc')] + ['--version'] - output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() - else: - compiler = "hipcc" - output = subprocess.run([compiler, "--version"], check=True, stdout=subprocess.PIPE).stdout.decode() - - for line in output.split('\n'): - if 'HIP version' in line: - globalParameters['HipClangVersion'] = line.split()[2] - print1("# Found hipcc version " + globalParameters['HipClangVersion']) - - except (subprocess.CalledProcessError, OSError) as e: - printWarning("Error: {} running {} {} ".format('hipcc', '--version', e)) - - # The following keys may be present in the config, but are not (or no longer) global parameters. - ignoreKeys = [ - "UseCompression", - "CxxCompiler", - "CCompiler", - "OffloadBundler", - "Assembler", - "LogicPath", - "LogicFilter", - "OutputPath", - "Experimental", - "GenSolTable" - ] - for key in config: - if key in ignoreKeys: - continue - value = config[key] - if key not in globalParameters: - printWarning("Global parameter %s = %s unrecognised." % ( key, value )) - globalParameters[key] = value -def setupRestoreClocks(): - import atexit - def restoreClocks(): - if globalParameters["PinClocks"]: - rsmi = globalParameters["ROCmSMIPath"] - subprocess.call([rsmi, "-d", "0", "--resetclocks"]) - subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) - atexit.register(restoreClocks) setupRestoreClocks() + ################################################################################ # Assign Parameters # populate dst with src[key] else give it the default/backup value ################################################################################ -def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, \ - defaultDictionary): - if key in sourceDictionary: - destinationDictionary[key] = deepcopy(sourceDictionary[key]) - else: - destinationDictionary[key] = deepcopy(defaultDictionary[key]) +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) + def ClientExecutionLock(): - if not globalParameters["ClientExecutionLockPath"]: - return open(os.devnull) + if not globalParameters["ClientExecutionLockPath"]: + return open(os.devnull) + + import filelock - import filelock - return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) + return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index 09aba3133b..2addea2cd1 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -1,10 +1,10 @@ from .Architectures import * -from .Utilities import * -from .GlobalParameters import * -from .Constants import * from .Capabilities import * -from .Parallel import * -from .Types import * +from .Constants import * # Dunder variables are not exported via `*` +from .GlobalParameters import * from .GlobalParameters import __version__ +from .Parallel import * +from .Types import * +from .Utilities import * diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 6f61d65591..73706d5c92 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,7 @@ from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Common import getGfxArch, internalParameters, globalParameters, state, state_key_ordering +from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering @state_key_ordering class FreeIndex: @@ -699,7 +699,7 @@ def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): if 'ISA' not in d: if d['KernelLanguage'] == 'Assembly': - d['ISA'] = getGfxArch(deviceInfo[1]) + d['ISA'] = gfxToIsa(deviceInfo[1]) else: d['ISA'] = [0,0,0] diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index 6e9bc57281..3b791d1c92 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -37,7 +37,7 @@ from . import ClientWriter from .Common import assignGlobalParameters, ensurePath, globalParameters, \ - printExit, getGfxName, getGfxCommonName + printExit, isaToGfx, gfxToSwCodename from .SolutionStructs import ProblemSizes from .Toolchain.Validators import ToolchainDefaults, validateToolchain @@ -68,8 +68,8 @@ def GenerateSummations(userArgs): cxxCompiler, cCompiler = validateToolchain(ToolchainDefaults.CXX_COMPILER, ToolchainDefaults.C_COMPILER) currentISA = globalParameters["CurrentISA"] - gfxName = getGfxName(currentISA) - commonName = getGfxCommonName(gfxName) + gfxName = isaToGfx(currentISA) + commonName = gfxToSwCodename(gfxName) globPath = os.path.join(inputLogicPath, "{}*".format(commonName)) logicFileNames = glob.glob(globPath) diff --git a/tensilelite/Tensile/Hardware.py b/tensilelite/Tensile/Hardware.py index dd888a6281..9f5c3c673b 100644 --- a/tensilelite/Tensile/Hardware.py +++ b/tensilelite/Tensile/Hardware.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,18 +23,18 @@ ################################################################################ from . import Properties -from .Common import getGfxName +from .Common import isaToGfx import copy class HardwarePredicate(Properties.Predicate): @classmethod def FromISA(cls, isa): - gfxArch = getGfxName(isa) + gfxArch = isaToGfx(isa) return cls("AMDGPU", value=cls("Processor", value=gfxArch)) @classmethod def FromHardware(cls, isa, cuCount=None): - gfxArch = getGfxName(isa) + gfxArch = isaToGfx(isa) if cuCount == None: return cls("AMDGPU", value=cls("Processor", value=gfxArch)) else: diff --git a/tensilelite/Tensile/KernelWriterActivationFunction.py b/tensilelite/Tensile/KernelWriterActivationFunction.py index 526178fe0c..e25a4ca905 100644 --- a/tensilelite/Tensile/KernelWriterActivationFunction.py +++ b/tensilelite/Tensile/KernelWriterActivationFunction.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,7 +23,7 @@ from copy import deepcopy from .TensileInstructions import TensileInstructions -from .Common import globalParameters, getGfxArch, getGfxName +from .Common import globalParameters, gfxToIsa, isaToGfx from .Activation import ActivationInline, ActivationType from .KernelWriterBase import KernelWriterBase @@ -55,7 +55,7 @@ def __init__(self, state, cxxCompiler: str): self.supportedArchs = deepcopy(globalParameters['SupportedISA']) else: for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = getGfxArch(''.join(map(str, arch))) + self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) # derive parameter self.language = "HIP" @@ -120,9 +120,9 @@ def getInlineAsm(self, activation: ActivationInline, spaces: int, activationType defineStr = [] macroStr = "#if" for archList in cateArch: - defStr = "%s defined(__%s__)"%(macroStr, getGfxName(archList[0])) + defStr = "%s defined(__%s__)"%(macroStr, isaToGfx(archList[0])) for arch in archList: - defStr += "|| defined(__%s__)"%getGfxName(arch) + defStr += "|| defined(__%s__)"%isaToGfx(arch) defStr += "\n" defineStr.append(defStr) macroStr = "#elif" diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 2755cd2274..582205c7cf 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, getGfxArch, getGfxName, INDEX_CHARS +from .Common import globalParameters, gfxToIsa, isaToGfx, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): @@ -88,7 +88,7 @@ def __init__(self, state, load_vw): self.supportedArchs = deepcopy(globalParameters['SupportedISA']) else: for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = getGfxArch(''.join(map(str, arch))) + self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) self.gsuKernels = [self.state["GlobalSplitU"]] if self.state["GenPGRPostKernels"]: @@ -538,9 +538,9 @@ def kernelBody(self): canPKF32Arch.append(arch) defineStr = [] if len(canPKF32Arch) > 0: - defineStr = "#if defined(__%s__)"%getGfxName(canPKF32Arch[0]) + defineStr = "#if defined(__%s__)"%isaToGfx(canPKF32Arch[0]) for arch in canPKF32Arch[1:]: - defineStr += "|| defined(__%s__)"%getGfxName(arch) + defineStr += "|| defined(__%s__)"%isaToGfx(arch) else: defineStr = "#if 0" # PGR=2 diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 7522b4c263..a266a5b4b9 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import getGfxArch, printExit, printWarning, print2, versionIsCompatible, __version__ +from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__ from typing import NamedTuple, List import os @@ -273,7 +273,7 @@ def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): # unpack solution def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: if solutionState["KernelLanguage"] == "Assembly": - solutionState["ISA"] = getGfxArch(data["ArchitectureName"]) + solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) else: solutionState["ISA"] = (0, 0, 0) solutionState["CUCount"] = data["CUCount"] diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index a56fb96205..79682e8c4f 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, getGfxArch, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -843,14 +843,14 @@ def meta_str(kernels: Tuple[KernelMeta]): debug_build: bool = args.debug_build arch: str = args.arch is_scale: bool = args.is_scale - isa = getGfxArch(arch) + isa = gfxToIsa(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) detectGlobalCurrentISA() isa = globalParameters['CurrentISA'] - arch = getGfxName(isa) + arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) ti.Base._global_ti.init(isa, toolchain_path, False) diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index f57bbbbcdd..2df20f2d17 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, getGfxArch, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -920,14 +920,14 @@ def meta_str(kernels: Tuple[KernelMeta]): toolchain_path: str = validateToolchain(args.toolchain) debug_build: bool = args.debug_build arch: str = args.arch - isa = getGfxArch(arch) + isa = gfxToIsa(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) detectGlobalCurrentISA() isa = globalParameters['CurrentISA'] - arch = getGfxName(isa) + arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) ti.Base._global_ti.init(isa, toolchain_path, False) diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index d771aa7ac1..34ec8c0cd2 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -32,7 +32,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, getGfxName, getGfxArch, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def record_num_calls(f): @@ -280,7 +280,7 @@ def global_read(self, srd_reg_idx: Union[int, str], module.add(ti.SWaitCnt(vmcnt=0)) return module, data_reg_idx - + def local_read(self, ext_local_byte_offset_reg_idx: Optional[int] = None, sync: bool = True): module = ti.Module() @@ -619,7 +619,7 @@ class KernelArgument: def to_dict(self): d = {'.size': self.size, '.offset': self.offset, '.value_kind': self.value_kind} - + if self.address_space: d['.address_space'] = self.address_space @@ -687,14 +687,14 @@ def meta_str(kernels: Tuple[KernelMeta]): toolchain_path: str = validateToolchain(args.toolchain) debug_build: bool = args.debug_build arch: str = args.arch - isa = getGfxArch(arch) + isa = gfxToIsa(arch) if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) detectGlobalCurrentISA() isa = globalParameters['CurrentISA'] - arch = getGfxName(isa) + arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) ti.Base._global_ti.init(isa, toolchain_path, False) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index a9a6fb7850..8fedd3396e 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -183,7 +183,7 @@ def FromOriginalState(cls, d, solutions): origTrees = d["trees"] trees = [] - + if "fallback" in d: fallbackIndex = d["fallback"] nullValue = SingleSolutionLibrary(solutions[fallbackIndex]) @@ -314,7 +314,7 @@ def hardware(d, problemType, solutions, library, placeholderName): if devicePart == "fallback": pred = Hardware.HardwarePredicate("TruePred") else: - pred = Hardware.HardwarePredicate.FromHardware(Common.getGfxArch(devicePart), cuCount) + pred = Hardware.HardwarePredicate.FromHardware(Common.gfxToIsa(devicePart), cuCount) newLib.rows.append({"predicate": pred, "library": library}) diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index 9e50528f6c..e7fd5b6ced 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -46,47 +46,149 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: "for a set of supplied logic files.", ) - argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") - argParser.add_argument("OutputPath", help="Where to write library files?") - argParser.add_argument("RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"]) - argParser.add_argument("--cxx-compiler", dest="CxxCompiler", action="store", default=ToolchainDefaults.CXX_COMPILER, - help=f"Default: {ToolchainDefaults.CXX_COMPILER}") - argParser.add_argument("--c-compiler", dest="CCompiler", action="store", default=ToolchainDefaults.C_COMPILER) - argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") - argParser.add_argument("--offload-bundler", dest="OffloadBundler", action="store", default=ToolchainDefaults.OFFLOAD_BUNDLER) - argParser.add_argument("--assembler", dest="Assembler", action="store", default=ToolchainDefaults.ASSEMBLER) - argParser.add_argument("--code-object-version", dest="CodeObjectVersion", choices=["4", "5"], default="4", action="store") - argParser.add_argument("--architecture", dest="Architecture", type=str, action="store", default="all", help="Supported archs: " + " ".join(architectureMap.keys())) - argParser.add_argument("--short-file-names", dest="ShortNames", action="store_true", default=False) - argParser.add_argument("--no-compress", dest="NoCompress", action="store_true", help="Don't compress assembly code objects.") - argParser.add_argument("--experimental", dest="Experimental", action="store_true", - help="Include logic files in directories named 'Experimental'.") - argParser.add_argument("--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator.") + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument("OutputPath", help="Where to write library files?") + argParser.add_argument( + "RuntimeLanguage", help="Which runtime language?", choices=["OCL", "HIP", "HSA"] + ) + argParser.add_argument( + "--cxx-compiler", + dest="CxxCompiler", + action="store", + default=ToolchainDefaults.CXX_COMPILER, + help=f"Default: {ToolchainDefaults.CXX_COMPILER}", + ) + argParser.add_argument( + "--c-compiler", dest="CCompiler", action="store", default=ToolchainDefaults.C_COMPILER + ) + argParser.add_argument("--cmake-cxx-compiler", dest="CmakeCxxCompiler", action="store") + argParser.add_argument( + "--offload-bundler", + dest="OffloadBundler", + action="store", + default=ToolchainDefaults.OFFLOAD_BUNDLER, + ) + argParser.add_argument( + "--assembler", dest="Assembler", action="store", default=ToolchainDefaults.ASSEMBLER + ) + argParser.add_argument( + "--code-object-version", + dest="CodeObjectVersion", + choices=["4", "5"], + default="4", + action="store", + ) + argParser.add_argument( + "--architecture", + dest="Architecture", + type=str, + action="store", + default="all", + help="Supported archs: " + " ".join(architectureMap.keys()), + ) + argParser.add_argument( + "--short-file-names", dest="ShortNames", action="store_true", default=False + ) + argParser.add_argument( + "--no-compress", + dest="NoCompress", + action="store_true", + help="Don't compress assembly code objects.", + ) + argParser.add_argument( + "--experimental", + dest="Experimental", + action="store_true", + help="Include logic files in directories named 'Experimental'.", + ) + argParser.add_argument( + "--no-enumerate", action="store_true", help="Do not run rocm_agent_enumerator." + ) argParser.add_argument("--version", help="Version string to embed into library file.") - argParser.add_argument("--logic-format", dest="LogicFormat", choices=["yaml", "json"], \ - action="store", default="yaml", help="select which logic format to use") - argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], - action="store", default="msgpack", help="select which library format to use") - argParser.add_argument("--jobs", "-j", dest="CpuThreads", type=int, - default=-1, help="Number of parallel jobs to launch.") - argParser.add_argument("--verbose", "-v", dest="PrintLevel", type=int, - default=1, help="Set printout verbosity level.") - argParser.add_argument("--no-lazy-library-loading", dest="LazyLibraryLoading", action="store_false", - default=True, help="Disable building for lazy library loading.") - argParser.add_argument("--enable-marker", dest="EnableMarker", action="store_true", - default=False, help="Enable marker in Tensile.") - argParser.add_argument("--no-generate-solution-table", dest="GenSolTable", action="store_false", default=True, - help="Skip generating solution-yaml matching table") - argParser.add_argument("--asm-debug", dest="AsmDebug", action="store_true", default=False, - help="Keep debug information for built code objects") + argParser.add_argument( + "--logic-format", + dest="LogicFormat", + choices=["yaml", "json"], + action="store", + default="yaml", + help="select which logic format to use", + ) + argParser.add_argument( + "--library-format", + dest="LibraryFormat", + choices=["yaml", "msgpack"], + action="store", + default="msgpack", + help="select which library format to use", + ) + argParser.add_argument( + "--jobs", + "-j", + dest="CpuThreads", + type=int, + default=-1, + help="Number of parallel jobs to launch.", + ) + argParser.add_argument( + "--verbose", + "-v", + dest="PrintLevel", + type=int, + default=1, + help="Set printout verbosity level.", + ) + argParser.add_argument( + "--no-lazy-library-loading", + dest="LazyLibraryLoading", + action="store_false", + default=True, + help="Disable building for lazy library loading.", + ) + argParser.add_argument( + "--enable-marker", + dest="EnableMarker", + action="store_true", + default=False, + help="Enable marker in Tensile.", + ) + argParser.add_argument( + "--no-generate-solution-table", + dest="GenSolTable", + action="store_false", + default=True, + help="Skip generating solution-yaml matching table", + ) + argParser.add_argument( + "--asm-debug", + dest="AsmDebug", + action="store_true", + default=False, + help="Keep debug information for built code objects", + ) argParser.add_argument("--build-id", dest="BuildIdKind", action="store", default="sha1") - argParser.add_argument("--address-sanitizer", dest="AsanBuild", action="store_true", - default=False, help="Enable ASAN build.") - argParser.add_argument("--keep-build-tmp", dest="KeepBuildTmp", action="store_true", - default=False, help="Do not remove the temporary build directory (may required hundreds of GBs of space)"), - argParser.add_argument("--logic-filter", dest="LogicFilter", action="store", default="*", type=str, - help="Cutomsized logic filter, default is *, i.e. all logics." - " Example: gfx942/Equality/* for building equality of gfx942 only") + argParser.add_argument( + "--address-sanitizer", + dest="AsanBuild", + action="store_true", + default=False, + help="Enable ASAN build.", + ) + argParser.add_argument( + "--keep-build-tmp", + dest="KeepBuildTmp", + action="store_true", + default=False, + help="Do not remove the temporary build directory (may required hundreds of GBs of space)", + ), + argParser.add_argument( + "--logic-filter", + dest="LogicFilter", + action="store", + default="*", + type=str, + help="Cutomsized logic filter, default is *, i.e. all logics." + " Example: gfx942/Equality/* for building equality of gfx942 only", + ) args = argParser.parse_args() @@ -97,13 +199,13 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: arguments["LazyLibraryLoading"] = args.LazyLibraryLoading arguments["EnableMarker"] = args.EnableMarker if args.CmakeCxxCompiler: - os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler + os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler arguments["ShortNames"] = args.ShortNames arguments["CodeFromFiles"] = False - arguments["LogicFormat"] = args.LogicFormat + arguments["LogicFormat"] = args.LogicFormat arguments["LibraryFormat"] = args.LibraryFormat if args.no_enumerate: - arguments["AMDGPUArchPath"] = False + arguments["AMDGPUArchPath"] = False arguments["CpuThreads"] = args.CpuThreads arguments["PrintLevel"] = args.PrintLevel arguments["AsmDebug"] = args.AsmDebug diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 444e9fd200..6bbc56b1e2 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -27,24 +27,44 @@ import itertools import os import shutil - from pathlib import Path from timeit import default_timer as timer -from typing import NamedTuple, List, Optional, Sequence, Union - -from Tensile import LibraryIO, SOURCE_PATH -from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles -from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles -from Tensile.Toolchain.Validators import validateToolchain, getVersion, ToolchainDefaults -from Tensile.TensileInstructions import TensileInstructions -from Tensile.Common import globalParameters, HR, print1, print2, printExit, IsaVersion, ensurePath, state, \ - CHeader, assignGlobalParameters, architectureMap, IsaVersion, ParallelMap2, tqdm, \ - SemanticVersion, getGfxName +from typing import List, NamedTuple, Optional, Sequence, Union + +from Tensile import SOURCE_PATH, LibraryIO +from Tensile.Common import ( + HR, + CHeader, + IsaVersion, + ParallelMap2, + SemanticVersion, + architectureMap, + assignGlobalParameters, + ensurePath, + isaToGfx, + globalParameters, + print1, + print2, + printExit, + state, + tqdm, +) +from Tensile.CustomYamlLoader import load_logic_gfx_arch from Tensile.KernelWriterAssembly import KernelWriterAssembly -from Tensile.KernelWriterBase import KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H +from Tensile.KernelWriterBase import ( + KERNEL_HELPER_FILENAME_CPP, + KERNEL_HELPER_FILENAME_H, +) from Tensile.SolutionLibrary import MasterSolutionLibrary from Tensile.SolutionStructs import Solution -from Tensile.CustomYamlLoader import load_logic_gfx_arch +from Tensile.TensileInstructions import TensileInstructions +from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles +from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles +from Tensile.Toolchain.Validators import ( + ToolchainDefaults, + getVersion, + validateToolchain, +) from Tensile.Utilities.Decorators.Profile import profile from Tensile.Utilities.Decorators.Timing import timing @@ -73,7 +93,9 @@ def processKernelSource(kernelWriterAssembly, ti, kernel) -> KernelCodeGenResult header = kernelWriter.getHeaderFileString(kernel) objFilename = kernel._state.get("codeObjectFile", None) - return KernelCodeGenResult(err, src, header, asmFilename, objFilename, tuple(kernel["ISA"]), kernel["WavefrontSize"]) + return KernelCodeGenResult( + err, src, header, asmFilename, objFilename, tuple(kernel["ISA"]), kernel["WavefrontSize"] + ) def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, globalParameters): @@ -82,10 +104,16 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, removeSolutions = [] removeResults = [] - for kernIdx, r in tqdm(enumerate(results)) if globalParameters["PrintLevel"] > 1 else enumerate(results): + for kernIdx, r in ( + tqdm(enumerate(results)) if globalParameters["PrintLevel"] > 1 else enumerate(results) + ): if r.err != 0: if not errorTolerant: - print("\nKernel generation failed for kernel: {}".format(kernels[kernIdx]["SolutionIndex"])) + print( + "\nKernel generation failed for kernel: {}".format( + kernels[kernIdx]["SolutionIndex"] + ) + ) print(kernels[kernIdx]["SolutionNameMin"]) removeKernels.append(kernels[kernIdx]) kName = Solution.getKeyNoInternalArgs(kernels[kernIdx]) @@ -99,7 +127,11 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, for kern in removeKernels: kernels.remove(kern) - for solution in tqdm(solutions, "Finding invalid solutions") if globalParameters["PrintLevel"] > 1 else solutions: + for solution in ( + tqdm(solutions, "Finding invalid solutions") + if globalParameters["PrintLevel"] > 1 + else solutions + ): solutionKernels = solution.getKernels() for kernel in solutionKernels: kName = Solution.getKeyNoInternalArgs(kernel) @@ -116,12 +148,12 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, def writeAssembly(asmPath: Union[Path, str], result: KernelCodeGenResult): if result.err: - printExit(f"Failed to build kernel {result.name} because it has error code {result.err}") + printExit(f"Failed to build kernel {result.name} because it has error code {result.err}") path = Path(asmPath) / f"{result.name}.s" - isa = result.isa + isa = result.isa wfsize = result.wavefrontSize with open(path, "w", encoding="utf-8") as f: - f.write(result.src) + f.write(result.src) # result.src is very large so let garbage collector know to clean up del result @@ -129,20 +161,23 @@ def writeAssembly(asmPath: Union[Path, str], result: KernelCodeGenResult): return path, isa, wfsize -def writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H): +def writeHelpers( + outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H +): kernelSourceFilename = os.path.join(os.path.normcase(outputPath), KERNEL_HELPER_FILENAME_CPP) kernelHeaderFilename = os.path.join(os.path.normcase(outputPath), KERNEL_HELPER_FILENAME_H) - with open(kernelHeaderFilename, "w", encoding="utf-8") as kernelHeaderFile, \ - open(kernelSourceFilename, "w", encoding="utf-8") as kernelSourceFile: + with open(kernelHeaderFilename, "w", encoding="utf-8") as kernelHeaderFile, open( + kernelSourceFilename, "w", encoding="utf-8" + ) as kernelSourceFile: kernelSourceFile.write(CHeader) kernelHeaderFile.write(CHeader) - kernelSourceFile.write("#include \"Kernels.h\"\n") + kernelSourceFile.write('#include "Kernels.h"\n') kernelHeaderFile.write("#pragma once\n") if globalParameters["RuntimeLanguage"] == "HIP": - kernelHeaderFile.write("#include \n") - kernelHeaderFile.write("#include \n\n") - kernelHeaderFile.write("#include \"KernelHeader.h\"\n\n") + kernelHeaderFile.write("#include \n") + kernelHeaderFile.write("#include \n\n") + kernelHeaderFile.write('#include "KernelHeader.h"\n\n') HeaderText = "" for ko in kernelHelperObjs: kernelName = ko.getKernelName() @@ -154,225 +189,295 @@ def writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNE kernelHeaderFile.write(HeaderText) -def writeSolutionsAndKernels(outputPath, asmToolchain, srcToolchain, solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, errorTolerant=False, generateSourcesAndExit=False, compress=True, fromTensile=False): - codeObjectFiles = [] - - outputPath = Path(outputPath) - destLibPath = ensurePath(outputPath / "library") # Destination for code object library files (.co) - buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) # - assemblyTmpPath = ensurePath(buildTmpPath / "assembly") # Temp path for generated assembly files (.s) - objectTmpPath = ensurePath(buildTmpPath / "code_object_tmp") # Temp path for HSA code object files (.hsaco) - - asmKernels = [k for k in kernels if k['KernelLanguage'] == 'Assembly'] - - visited = set() - duplicates = 0 - for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) - k.duplicate = True if base in visited else False - duplicates += k.duplicate - print2(f"Duplicate: {base}") - visited.add(base) - print1(f"Number of duplicate kernels: {duplicates}") - - numAsmKernels = len(asmKernels) - numKernels = len(asmKernels) - assert numKernels == numAsmKernels, "Only assembly kernels are supported in TensileLite" - asmIter = zip(itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), asmKernels) - asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels") - removeInvalidSolutionsAndKernels(asmResults, asmKernels, solutions, errorTolerant, globalParameters) - def assemble(ret): - p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), getGfxName(isa), wavefrontsize) - unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) - compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) - ret = ParallelMap2(compose(assemble, unaryWriteAssembly), asmResults, "Writing assembly kernels", return_as="list", multiArg=False) - - writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) - srcKernelFile = Path(outputPath) / "Kernels.cpp" - - if not generateSourcesAndExit: - codeObjectFiles += buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress) - buildSourceCodeObjectFiles(srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile) - - return codeObjectFiles, numKernels - - -def writeSolutionsAndKernelsTCL(outputPath, asmToolchain, srcToolchain, kernels, kernelHelperObjs, \ - kernelWriterAssembly, compress=True, fromTensile=False): - - outputPath = Path(outputPath) - destLibPath = ensurePath(outputPath / "library") # Destination for code object library files (.co) - buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) - assemblyTmpPath = ensurePath(buildTmpPath / "assembly") # Temp path for generated assembly files (.s) - objectTmpPath = ensurePath(buildTmpPath / "code_object_tmp") # Temp path for HSA code object files (.hsaco) - - asmKernels = [k for k in kernels if k['KernelLanguage'] == 'Assembly'] - - visited = set() - duplicates = 0 - for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) - k.duplicate = True if base in visited else False - duplicates += k.duplicate - print2(f"Duplicate: {base}") - visited.add(base) - print1(f"Number of duplicate kernels: {duplicates}") - - uniqueAsmKernels = [k for k in asmKernels if not k.duplicate] - def assemble(ret): - p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), getGfxName(isa), wavefrontsize) - unaryProcessKernelSource = functools.partial(processKernelSource, kernelWriterAssembly, TensileInstructions()) - unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) - compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) - ret = ParallelMap2(compose(assemble, unaryWriteAssembly, unaryProcessKernelSource), uniqueAsmKernels, "Generating assembly kernels", multiArg=False) - buildAssemblyCodeObjectFiles(asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress) - - writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) - srcKernelFile = Path(outputPath) / "Kernels.cpp" - buildSourceCodeObjectFiles(srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile) - - return len(uniqueAsmKernels) +def writeSolutionsAndKernels( + outputPath, + asmToolchain, + srcToolchain, + solutions, + kernels, + kernelHelperObjs, + kernelWriterAssembly, + errorTolerant=False, + generateSourcesAndExit=False, + compress=True, + fromTensile=False, +): + codeObjectFiles = [] + + outputPath = Path(outputPath) + destLibPath = ensurePath( + outputPath / "library" + ) # Destination for code object library files (.co) + buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) # + assemblyTmpPath = ensurePath( + buildTmpPath / "assembly" + ) # Temp path for generated assembly files (.s) + objectTmpPath = ensurePath( + buildTmpPath / "code_object_tmp" + ) # Temp path for HSA code object files (.hsaco) + + asmKernels = [k for k in kernels if k["KernelLanguage"] == "Assembly"] + + visited = set() + duplicates = 0 + for k in asmKernels: + base = kernelWriterAssembly.getKernelFileBase(k) + k.duplicate = True if base in visited else False + duplicates += k.duplicate + print2(f"Duplicate: {base}") + visited.add(base) + print1(f"Number of duplicate kernels: {duplicates}") + + numAsmKernels = len(asmKernels) + numKernels = len(asmKernels) + assert numKernels == numAsmKernels, "Only assembly kernels are supported in TensileLite" + asmIter = zip( + itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), asmKernels + ) + asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels") + removeInvalidSolutionsAndKernels( + asmResults, asmKernels, solutions, errorTolerant, globalParameters + ) + + def assemble(ret): + p, isa, wavefrontsize = ret + asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + + unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) + compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) + ret = ParallelMap2( + compose(assemble, unaryWriteAssembly), + asmResults, + "Writing assembly kernels", + return_as="list", + multiArg=False, + ) + + writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) + srcKernelFile = Path(outputPath) / "Kernels.cpp" + + if not generateSourcesAndExit: + codeObjectFiles += buildAssemblyCodeObjectFiles( + asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + ) + buildSourceCodeObjectFiles( + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + ) + + return codeObjectFiles, numKernels + + +def writeSolutionsAndKernelsTCL( + outputPath, + asmToolchain, + srcToolchain, + kernels, + kernelHelperObjs, + kernelWriterAssembly, + compress=True, + fromTensile=False, +): + + outputPath = Path(outputPath) + destLibPath = ensurePath( + outputPath / "library" + ) # Destination for code object library files (.co) + buildTmpPath = ensurePath(outputPath / "build_tmp" / outputPath.stem.upper()) + assemblyTmpPath = ensurePath( + buildTmpPath / "assembly" + ) # Temp path for generated assembly files (.s) + objectTmpPath = ensurePath( + buildTmpPath / "code_object_tmp" + ) # Temp path for HSA code object files (.hsaco) + + asmKernels = [k for k in kernels if k["KernelLanguage"] == "Assembly"] + + visited = set() + duplicates = 0 + for k in asmKernels: + base = kernelWriterAssembly.getKernelFileBase(k) + k.duplicate = True if base in visited else False + duplicates += k.duplicate + print2(f"Duplicate: {base}") + visited.add(base) + print1(f"Number of duplicate kernels: {duplicates}") + + uniqueAsmKernels = [k for k in asmKernels if not k.duplicate] + + def assemble(ret): + p, isa, wavefrontsize = ret + asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + + unaryProcessKernelSource = functools.partial( + processKernelSource, kernelWriterAssembly, TensileInstructions() + ) + unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) + compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) + ret = ParallelMap2( + compose(assemble, unaryWriteAssembly, unaryProcessKernelSource), + uniqueAsmKernels, + "Generating assembly kernels", + multiArg=False, + ) + buildAssemblyCodeObjectFiles( + asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + ) + + writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) + srcKernelFile = Path(outputPath) / "Kernels.cpp" + buildSourceCodeObjectFiles( + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + ) + + return len(uniqueAsmKernels) @timing -def getSolutionAndKernelWriters(solutions, kernels, assembler: str, assemblerVersion: SemanticVersion): - kernelSerialNaming = Solution.getSerialNaming(kernels) - solutionMinNaming = Solution.getMinNaming(solutions) - kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion) +def getSolutionAndKernelWriters( + solutions, kernels, assembler: str, assemblerVersion: SemanticVersion +): + kernelSerialNaming = Solution.getSerialNaming(kernels) + solutionMinNaming = Solution.getMinNaming(solutions) + kernelMinNaming = Solution.getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly( + kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion + ) - return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) + return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) @timing def copyStaticFiles(outputPath): - libraryStaticFiles = [ - "TensileTypes.h", - "tensile_bfloat16.h", - "tensile_float8_bfloat8.h", - "tensile_float8_bfloat8_bc.h", - "KernelHeader.h", - "ReductionTemplate.h", - "memory_gfx.h" ] + libraryStaticFiles = [ + "TensileTypes.h", + "tensile_bfloat16.h", + "tensile_float8_bfloat8.h", + "tensile_float8_bfloat8_bc.h", + "KernelHeader.h", + "ReductionTemplate.h", + "memory_gfx.h", + ] - for fileName in libraryStaticFiles: - shutil.copy(os.path.join(SOURCE_PATH, fileName), outputPath) + for fileName in libraryStaticFiles: + shutil.copy(os.path.join(SOURCE_PATH, fileName), outputPath) - return libraryStaticFiles + return libraryStaticFiles @timing def generateKernelObjectsFromSolutions(solutions): - kernels = [] - kernelHelperObjs = [] - kernelNames = set() - kernelHelperNames = set() + kernels = [] + kernelHelperObjs = [] + kernelNames = set() + kernelHelperNames = set() - for solution in solutions: - solutionKernels = solution.getKernels() - for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) - if kName not in kernelNames: - kernels.append(kernel) - kernelNames.add(kName) - solutionHelperKernels = solution.getHelperKernelObjects() - kernelHelperObjs += solutionHelperKernels - for ko in solutionHelperKernels: - kernelHelperNames.add(ko.getKernelName()) + for solution in solutions: + solutionKernels = solution.getKernels() + for kernel in solutionKernels: + kName = Solution.getKeyNoInternalArgs(kernel) + if kName not in kernelNames: + kernels.append(kernel) + kernelNames.add(kName) + solutionHelperKernels = solution.getHelperKernelObjects() + kernelHelperObjs += solutionHelperKernels + for ko in solutionHelperKernels: + kernelHelperNames.add(ko.getKernelName()) - # remove duplicates while preserving order - numKhos = len(kernelHelperObjs) - kernelHelperObjs = list(dict.fromkeys(kernelHelperObjs)) + # remove duplicates while preserving order + numKhos = len(kernelHelperObjs) + kernelHelperObjs = list(dict.fromkeys(kernelHelperObjs)) - print1(f"Number of kernel helper objects: {numKhos}") - print1(f"Number of unique kernel helper objects: {len(kernelHelperObjs)}") + print1(f"Number of kernel helper objects: {numKhos}") + print1(f"Number of unique kernel helper objects: {len(kernelHelperObjs)}") - return (kernels, kernelHelperObjs, kernelHelperNames) + return (kernels, kernelHelperObjs, kernelHelperNames) @timing def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): - if ";" in args["Architecture"]: - archs = args["Architecture"].split(";") # user arg list format - else: - archs = args["Architecture"].split("_") # workaround for cmake list in list issue - - solutions = [] - masterLibraries = {} - nextSolIndex = 0 - - fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(archs)) - - def libraryIter(lib: MasterSolutionLibrary): - if len(lib.solutions): - for i, s in enumerate(lib.solutions.items()): - yield (i, *s) + if ";" in args["Architecture"]: + archs = args["Architecture"].split(";") # user arg list format else: - for _, lazyLib in lib.lazyLibraries.items(): - yield from libraryIter(lazyLib) + archs = args["Architecture"].split("_") # workaround for cmake list in list issue + + solutions = [] + masterLibraries = {} + nextSolIndex = 0 + + fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(archs)) + + def libraryIter(lib: MasterSolutionLibrary): + if len(lib.solutions): + for i, s in enumerate(lib.solutions.items()): + yield (i, *s) + else: + for _, lazyLib in lib.lazyLibraries.items(): + yield from libraryIter(lazyLib) + + for library in ParallelMap2( + LibraryIO.parseLibraryLogicFile, fIter, "Loading Logics...", return_as="generator_unordered" + ): + _, architectureName, _, _, _, newLibrary = library + + if architectureName == "": + continue + + if architectureName in masterLibraries: + nextSolIndex = masterLibraries[architectureName].merge(newLibrary, nextSolIndex) + else: + masterLibraries[architectureName] = newLibrary + masterLibraries[architectureName].version = args["CodeObjectVersion"] + + # Sort masterLibraries to make global soln index values deterministic + solnReIndex = 0 + masterLibraries = dict(sorted(masterLibraries.items())) + for k, v in masterLibraries.items(): + for _, masterLibrary in masterLibraries.items(): + for _, sol in masterLibrary.solutions.items(): + sol.index = solnReIndex + solnReIndex += 1 + # Sort masterLibrary to make global soln index values deterministic + masterLibrary.lazyLibraries = dict(sorted(masterLibrary.lazyLibraries.items())) + for name, lib in masterLibrary.lazyLibraries.items(): + # Sort solns by the lib logic file they were generated from + lib.solutions = { + k: lib.solutions[k] + for k in sorted(lib.solutions, key=lambda idx: lib.solutions[idx].srcName) + } + for _, sol in lib.solutions.items(): + sol.index = solnReIndex + solnReIndex += 1 + + if args["GenSolTable"]: + matchTable = {} + # Match yaml file solutions to solution index + for _, masterLibrary in masterLibraries.items(): + for localIdx, _, s in libraryIter(masterLibrary): + matchTable[s.index] = [s.srcName, localIdx] + LibraryIO.write("MatchTable", matchTable) + + if "fallback" in masterLibraries.keys(): + for key, value in masterLibraries.items(): + if key != "fallback": + value.merge(masterLibraries["fallback"]) + masterLibraries.pop("fallback") + for _, masterLibrary in masterLibraries.items(): + for _, sol in masterLibrary.solutions.items(): + solutions.append(sol.originalSolution) + for name, lib in masterLibrary.lazyLibraries.items(): + for _, sol in lib.solutions.items(): + sol.originalSolution._state["codeObjectFile"] = name + solutions.append(sol.originalSolution) - for library in ParallelMap2(LibraryIO.parseLibraryLogicFile, fIter, "Loading Logics...", return_as="generator_unordered"): - _, architectureName, _, _, _, newLibrary = library + # remove duplicates while preserving order + numSoln = len(solutions) + solutions = dict.fromkeys(solutions).keys() - if architectureName == "": - continue + print1(f"Number of solutions parsed: {numSoln}") + print1(f"Number of unique solutions: {len(solutions)}") - if architectureName in masterLibraries: - nextSolIndex = masterLibraries[architectureName].merge(newLibrary, nextSolIndex) - else: - masterLibraries[architectureName] = newLibrary - masterLibraries[architectureName].version = args["CodeObjectVersion"] - - # Sort masterLibraries to make global soln index values deterministic - solnReIndex=0 - masterLibraries = dict(sorted(masterLibraries.items())) - for k,v in masterLibraries.items(): - for _, masterLibrary in masterLibraries.items(): - for _, sol in masterLibrary.solutions.items(): - sol.index = solnReIndex - solnReIndex += 1 - # Sort masterLibrary to make global soln index values deterministic - masterLibrary.lazyLibraries = dict(sorted(masterLibrary.lazyLibraries.items())) - for name, lib in masterLibrary.lazyLibraries.items(): - # Sort solns by the lib logic file they were generated from - lib.solutions = {k: lib.solutions[k] for k in sorted(lib.solutions, key = lambda idx: lib.solutions[idx].srcName )} - for _, sol in lib.solutions.items(): - sol.index = solnReIndex - solnReIndex += 1 - - if args["GenSolTable"]: - matchTable = {} - # Match yaml file solutions to solution index - for _,masterLibrary in masterLibraries.items(): - for localIdx, _, s in libraryIter(masterLibrary): - matchTable[s.index] = [s.srcName, localIdx] - LibraryIO.write("MatchTable", matchTable) - - if "fallback" in masterLibraries.keys(): - for key, value in masterLibraries.items(): - if key != "fallback": - value.merge(masterLibraries["fallback"]) - masterLibraries.pop("fallback") - for _, masterLibrary in masterLibraries.items(): - for _, sol in masterLibrary.solutions.items(): - solutions.append(sol.originalSolution) - for name, lib in masterLibrary.lazyLibraries.items(): - for _, sol in lib.solutions.items(): - sol.originalSolution._state["codeObjectFile"] = name - solutions.append(sol.originalSolution) - - # remove duplicates while preserving order - numSoln = len(solutions) - solutions = dict.fromkeys(solutions).keys() - - print1(f"Number of solutions parsed: {numSoln}") - print1(f"Number of unique solutions: {len(solutions)}") - - return solutions, masterLibraries + return solutions, masterLibraries ################################################################################ @@ -380,119 +485,148 @@ def libraryIter(lib: MasterSolutionLibrary): ################################################################################ @profile def run(): - start = timer() - print1("") - print1(HR) - print1("# Tensile Create Library") - print2(HR) - print2("") - - arguments = parseArguments() - outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) - cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( - arguments["CxxCompiler"], - arguments["CCompiler"], - arguments["OffloadBundler"], - arguments["Assembler"], - ToolchainDefaults.HIP_CONFIG, - ) - print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") - print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") - print1(f"# C Compiler: {cCompiler} (version {getVersion(cCompiler)})") - print1(f"# Assembler: {assembler} (version {getVersion(assembler)})") - print1(f"# Offload Bundler: {offloadBundler} (version {getVersion(offloadBundler)})") - print1(f"# Code Object Version: {arguments['CodeObjectVersion']}") - print1(f"# Architecture(s): {arguments['Architecture']}") - print1(f"# Library Format: {arguments['LibraryFormat']}") - - assignGlobalParameters(arguments, cxxCompiler) - - asmToolchain = AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"]) - srcToolchain = SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) - - if not os.path.exists(arguments["LogicPath"]): - printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") - - if ";" in arguments["Architecture"]: - archs = arguments["Architecture"].split(";") - else: - archs = arguments["Architecture"].split("_") - logicArchs = set() - for arch in archs: - if arch in architectureMap: - logicArchs.add(architectureMap[arch]) + start = timer() + print1("") + print1(HR) + print1("# Tensile Create Library") + print2(HR) + print2("") + + arguments = parseArguments() + outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) + cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( + arguments["CxxCompiler"], + arguments["CCompiler"], + arguments["OffloadBundler"], + arguments["Assembler"], + ToolchainDefaults.HIP_CONFIG, + ) + print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") + print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") + print1(f"# C Compiler: {cCompiler} (version {getVersion(cCompiler)})") + print1(f"# Assembler: {assembler} (version {getVersion(assembler)})") + print1(f"# Offload Bundler: {offloadBundler} (version {getVersion(offloadBundler)})") + print1(f"# Code Object Version: {arguments['CodeObjectVersion']}") + print1(f"# Architecture(s): {arguments['Architecture']}") + print1(f"# Library Format: {arguments['LibraryFormat']}") + + assignGlobalParameters(arguments, cxxCompiler) + + asmToolchain = AssemblyToolchain( + assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"] + ) + srcToolchain = SourceToolchain( + cxxCompiler, + offloadBundler, + globalParameters["BuildIdKind"], + globalParameters["AsanBuild"], + globalParameters["SaveTemps"], + ) + + if not os.path.exists(arguments["LogicPath"]): + printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") + + if ";" in arguments["Architecture"]: + archs = arguments["Architecture"].split(";") else: - printExit("Architecture %s not supported" % arch) - - logicExtFormat = ".yaml" - if arguments["LogicFormat"] == "yaml": - pass - elif arguments["LogicFormat"] == "json": - logicExtFormat = ".json" - else: - printExit(f"Unrecognized LogicFormat: {arguments['LogicFormat']}") - - def archMatch(arch: str, archs: List[str]): - return (arch in archs) or any(a.startswith(arch) for a in archs) - - def validLogicFile(p: Path): - return p.suffix == logicExtFormat and ("all" in archs or archMatch(load_logic_gfx_arch(p), archs)) - - globPattern = os.path.join(arguments["LogicPath"], f"**/{arguments['LogicFilter']}{logicExtFormat}") - print1(f"# LogicFilter: {globPattern}") - logicFiles = (os.path.join(arguments["LogicPath"], file) for file in glob.iglob(globPattern, recursive=True)) - logicFiles = [file for file in logicFiles if validLogicFile(Path(file))] - - print1(f"# Experimental: {arguments['Experimental']}") - if not arguments["Experimental"]: - logicFiles = [file for file in logicFiles if "experimental" not in map(str.lower, Path(file).parts)] - - print2(f"# LibraryLogicFiles: {len(logicFiles)}") - for logicFile in logicFiles: - print2("# %s" % logicFile) - - solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) - kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) - kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters(solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion) - - copyStaticFiles(outputPath) - - numKernels = writeSolutionsAndKernelsTCL(outputPath, asmToolchain, srcToolchain, kernels, - kernelHelperObjs, kernelWriterAssembly, compress=arguments["UseCompression"]) - - archs = [getGfxName(arch) for arch in globalParameters['SupportedISA'] \ - if globalParameters["AsmCaps"][arch]["SupportedISA"]] - newLibraryDir = ensurePath(os.path.join(outputPath, 'library')) - - for archName, newMasterLibrary in masterLibraries.items(): - if archName in archs: - if globalParameters["LazyLibraryLoading"]: - masterFile = os.path.join(newLibraryDir, "TensileLibrary_lazy_"+archName) - else: - masterFile = os.path.join(newLibraryDir, "TensileLibrary_"+archName) - newMasterLibrary.applyNaming(kernelMinNaming) - LibraryIO.write(masterFile, state(newMasterLibrary), arguments["LibraryFormat"]) - for name, lib in newMasterLibrary.lazyLibraries.items(): - filename = os.path.join(newLibraryDir, name) - lib.applyNaming(kernelMinNaming) - LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) - - if not globalParameters["KeepBuildTmp"]: - buildTmp = Path(arguments["OutputPath"]).parent / "library" / "build_tmp" - if buildTmp.exists() and buildTmp.is_dir(): - shutil.rmtree(buildTmp) - buildTmp = Path(arguments["OutputPath"]) / "build_tmp" - if buildTmp.exists() and buildTmp.is_dir(): - shutil.rmtree(buildTmp) + archs = arguments["Architecture"].split("_") + logicArchs = set() + for arch in archs: + if arch in architectureMap: + logicArchs.add(architectureMap[arch]) + else: + printExit("Architecture %s not supported" % arch) + + logicExtFormat = ".yaml" + if arguments["LogicFormat"] == "yaml": + pass + elif arguments["LogicFormat"] == "json": + logicExtFormat = ".json" else: - printWarning(f"Cannot remove build_tmp") - - print1("# Tensile Library Writer DONE") - print1(HR) - print1("") - - stop = timer() - - print1(f"Total time (s): {(stop-start):3.2f}") - print1(f"Total kernels processed: {numKernels}") - print1(f"Kernels processed per second: {(numKernels/(stop-start)):3.2f}") + printExit(f"Unrecognized LogicFormat: {arguments['LogicFormat']}") + + def archMatch(arch: str, archs: List[str]): + return (arch in archs) or any(a.startswith(arch) for a in archs) + + def validLogicFile(p: Path): + return p.suffix == logicExtFormat and ( + "all" in archs or archMatch(load_logic_gfx_arch(p), archs) + ) + + globPattern = os.path.join( + arguments["LogicPath"], f"**/{arguments['LogicFilter']}{logicExtFormat}" + ) + print1(f"# LogicFilter: {globPattern}") + logicFiles = ( + os.path.join(arguments["LogicPath"], file) + for file in glob.iglob(globPattern, recursive=True) + ) + logicFiles = [file for file in logicFiles if validLogicFile(Path(file))] + + print1(f"# Experimental: {arguments['Experimental']}") + if not arguments["Experimental"]: + logicFiles = [ + file for file in logicFiles if "experimental" not in map(str.lower, Path(file).parts) + ] + + print2(f"# LibraryLogicFiles: {len(logicFiles)}") + for logicFile in logicFiles: + print2("# %s" % logicFile) + + solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) + kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) + kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( + solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion + ) + + copyStaticFiles(outputPath) + + numKernels = writeSolutionsAndKernelsTCL( + outputPath, + asmToolchain, + srcToolchain, + kernels, + kernelHelperObjs, + kernelWriterAssembly, + compress=arguments["UseCompression"], + ) + + archs = [ + isaToGfx(arch) + for arch in globalParameters["SupportedISA"] + if globalParameters["AsmCaps"][arch]["SupportedISA"] + ] + newLibraryDir = ensurePath(os.path.join(outputPath, "library")) + + for archName, newMasterLibrary in masterLibraries.items(): + if archName in archs: + if globalParameters["LazyLibraryLoading"]: + masterFile = os.path.join(newLibraryDir, "TensileLibrary_lazy_" + archName) + else: + masterFile = os.path.join(newLibraryDir, "TensileLibrary_" + archName) + newMasterLibrary.applyNaming(kernelMinNaming) + LibraryIO.write(masterFile, state(newMasterLibrary), arguments["LibraryFormat"]) + for name, lib in newMasterLibrary.lazyLibraries.items(): + filename = os.path.join(newLibraryDir, name) + lib.applyNaming(kernelMinNaming) + LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) + + if not globalParameters["KeepBuildTmp"]: + buildTmp = Path(arguments["OutputPath"]).parent / "library" / "build_tmp" + if buildTmp.exists() and buildTmp.is_dir(): + shutil.rmtree(buildTmp) + buildTmp = Path(arguments["OutputPath"]) / "build_tmp" + if buildTmp.exists() and buildTmp.is_dir(): + shutil.rmtree(buildTmp) + else: + printWarning(f"Cannot remove build_tmp") + + print1("# Tensile Library Writer DONE") + print1(HR) + print1("") + + stop = timer() + + print1(f"Total time (s): {(stop-start):3.2f}") + print1(f"Total kernels processed: {numKernels}") + print1(f"Kernels processed per second: {(numKernels/(stop-start)):3.2f}") diff --git a/tensilelite/Tensile/TensileCreateLibrary/__init__.py b/tensilelite/Tensile/TensileCreateLibrary/__init__.py index a7cced5f5a..eb1674916c 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/__init__.py +++ b/tensilelite/Tensile/TensileCreateLibrary/__init__.py @@ -1,3 +1 @@ -from .Run import run -from .Run import copyStaticFiles -from .Run import writeSolutionsAndKernels +from .Run import copyStaticFiles, run, writeSolutionsAndKernels diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index 426986e266..21511588fe 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -188,4 +188,4 @@ def getSlcBitName(hasGLCModifier): def _removeIdent(isaDict) -> list: ids = [th.ident for th in threading.enumerate()] isaDict = [id for id in isaDict if id in ids] - return isaDict \ No newline at end of file + return isaDict diff --git a/tensilelite/Tensile/TensileInstructions/Code.py b/tensilelite/Tensile/TensileInstructions/Code.py index fbf450fa90..92b1abd84d 100644 --- a/tensilelite/Tensile/TensileInstructions/Code.py +++ b/tensilelite/Tensile/TensileInstructions/Code.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..Common import getGfxName +from ..Common import isaToGfx from .Base import Item from .Enums import SignatureValueKind from .Formatting import slash, slash50, block, block3Line, blockNewLine, \ @@ -694,7 +694,7 @@ def setGprs(self, totalVgprs: int, totalAgprs: int, totalSgprs: int): def getNextFreeVgpr(self) -> int: return self.totalVgprs - + def getNextFreeSgpr(self) -> int: return self.totalSgprs @@ -702,7 +702,7 @@ def __str__(self): kdIndent = " " * 2 kStr = "" kStr += ".amdgcn_target \"amdgcn-amd-amdhsa--%s\"\n" \ - % (getGfxName(self.kernel.isa)) + % (isaToGfx(self.kernel.isa)) kStr += ".text\n" kStr += ".protected %s\n" % self.name kStr += ".globl %s\n" % self.name @@ -899,7 +899,7 @@ def setGprs(self, totalVgprs: int, totalAgprs: int, totalSgprs: int): self.totalSgprs = totalSgprs self.signature.setGprs(totalVgprs=totalVgprs, totalAgprs=totalAgprs, \ totalSgprs=totalSgprs) - + def getNextFreeVgpr(self) -> int: return self.signature.getNextFreeVgpr() diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index e3b9509472..f009c63650 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -32,7 +32,7 @@ from pathlib import Path from typing import List, Union -from ..Common import globalParameters, print2, ensurePath, SemanticVersion, getGfxName +from ..Common import globalParameters, print2, ensurePath, SemanticVersion, isaToGfx from ..KernelWriterAssembly import KernelWriterAssembly from ..Toolchain.Validators import getVersion from ..SolutionStructs import Solution @@ -206,7 +206,7 @@ def buildAssemblyCodeObjectFiles( if len(archKernels) == 0: continue - gfx = getGfxName(arch) + gfx = isaToGfx(arch) objectFiles = [str(asmDir / (writer.getKernelFileBase(k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] coFileMap = collections.defaultdict(list) diff --git a/tensilelite/Tensile/Utilities/Decorators/Profile.py b/tensilelite/Tensile/Utilities/Decorators/Profile.py index 5439e2a8c5..6918bde64f 100644 --- a/tensilelite/Tensile/Utilities/Decorators/Profile.py +++ b/tensilelite/Tensile/Utilities/Decorators/Profile.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2016-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2016-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,23 +24,24 @@ import cProfile import pstats - -from pathlib import Path from datetime import datetime, timezone +from pathlib import Path from typing import Callable, Tuple from .Shared import envVariableIsSet PROFILE_ENV_VAR = "TENSILE_PROFILE" + def profile(func: Callable) -> Callable: - """Profiling decorator. + """Profiling decorator. - Add ``@profile`` to mark a function for profiling; set the environment variable + Add ``@profile`` to mark a function for profiling; set the environment variable TENSILE_PROFILE=ON to enable profiling decorated functions. """ if not envVariableIsSet(PROFILE_ENV_VAR): return func + def wrapper(*args, **kwargs): path, filename = initProfileArtifacts(func.__name__) @@ -48,22 +49,23 @@ def wrapper(*args, **kwargs): output = prof.runcall(func, *args, **kwargs) result = pstats.Stats(prof) result.sort_stats(pstats.SortKey.TIME) - result.dump_stats(path/filename) + result.dump_stats(path / filename) return output + return wrapper def initProfileArtifacts(funcName: str) -> Tuple[Path, str]: """Initializes filenames and paths for profiling artifacts based on the current datetime Args: - funcName: The name of the function being profiled, nominally passed via func.__name__ + funcName: The name of the function being profiled, nominally passed via func.__name__ Returns: A tuple (path, filename) where the path is the artifact directory and filename is a .prof file with the profiling results. """ dt = datetime.now(timezone.utc) filename = f"{funcName}-{dt.strftime('%Y-%m-%dT%H-%M-%SZ')}.prof" - path = Path().cwd()/f"profiling-results-{dt.strftime('%Y-%m-%d')}" + path = Path().cwd() / f"profiling-results-{dt.strftime('%Y-%m-%d')}" path.mkdir(exist_ok=True) return path, filename diff --git a/tensilelite/Tensile/Utilities/Decorators/Shared.py b/tensilelite/Tensile/Utilities/Decorators/Shared.py index b87f0fbfc8..aaa77046bf 100644 --- a/tensilelite/Tensile/Utilities/Decorators/Shared.py +++ b/tensilelite/Tensile/Utilities/Decorators/Shared.py @@ -24,6 +24,7 @@ import os + def envVariableIsSet(varName: str) -> bool: """Checks if the provided environment variable is set to "YES", "ON", "TRUE", or "1" Args: @@ -32,4 +33,4 @@ def envVariableIsSet(varName: str) -> bool: True if the environment variable is set, otherwise False. """ value = os.environ.get(varName, "").upper() - return True if value in ["YES", "ON", "TRUE", "1"] else False \ No newline at end of file + return True if value in ["YES", "ON", "TRUE", "1"] else False diff --git a/tensilelite/Tensile/Utilities/Decorators/Timing.py b/tensilelite/Tensile/Utilities/Decorators/Timing.py index 475199b0b6..f178937498 100644 --- a/tensilelite/Tensile/Utilities/Decorators/Timing.py +++ b/tensilelite/Tensile/Utilities/Decorators/Timing.py @@ -22,25 +22,28 @@ # ################################################################################ -from typing import Callable from timeit import default_timer as timer +from typing import Callable from .Shared import envVariableIsSet TIMING_ENV_VAR: str = "TENSILE_PRINT_TIMING" + def timing(func: Callable) -> Callable: - f"""Timing decorator to measure execution time of a function. + f"""Timing decorator to measure execution time of a function. Add ``@timing`` to mark a function for timing; set the environment variable {TIMING_ENV_VAR}=ON to enable timing decorated functions. """ - if not envVariableIsSet(TIMING_ENV_VAR): - return func - def wrapper(*args, **kwargs): - start = timer() - res = func(*args, **kwargs) - end = timer() - print(f'{func.__name__} took {end - start} seconds') - return res - return wrapper \ No newline at end of file + if not envVariableIsSet(TIMING_ENV_VAR): + return func + + def wrapper(*args, **kwargs): + start = timer() + res = func(*args, **kwargs) + end = timer() + print(f"{func.__name__} took {end - start} seconds") + return res + + return wrapper diff --git a/tensilelite/Tensile/__init__.py b/tensilelite/Tensile/__init__.py index 102dfe3d05..1266edcf59 100644 --- a/tensilelite/Tensile/__init__.py +++ b/tensilelite/Tensile/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,8 +26,7 @@ from __future__ import print_function from os import path -# Hardcoded tensilelite version -# Also in Tensile/Source/TensileConfigVersion.cmake +# Hardcoded tensilelite version, also in Tensile/Source/TensileConfigVersion.cmake __version__ = "4.33.0" ROOT_PATH: str = path.dirname(__file__) @@ -37,4 +36,4 @@ def PrintTensileRoot(): print(ROOT_PATH, end='') -__all__ = ["__version__", "ROOT_PATH", "SOURCE_PATH"] \ No newline at end of file +__all__ = ["__version__", "ROOT_PATH", "SOURCE_PATH", "CUSTOM_KERNEL_PATH"] diff --git a/tensilelite/Tensile/bin/TensileValidateLogic b/tensilelite/Tensile/bin/TensileValidateLogic deleted file mode 100755 index 92cc514388..0000000000 --- a/tensilelite/Tensile/bin/TensileValidateLogic +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 - -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -# This script only gets called by CMake - -try: - from Tensile import TensileValidateLogic -except ImportError: - import os.path - import sys - parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) - print(parentdir) - sys.path.append(parentdir) - - from Tensile import TensileValidateLogic - -################################################################################ -# Main -################################################################################ -if __name__ == "__main__": - TensileValidateLogic.run() diff --git a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake index a70d403c64..13e3b920e4 100644 --- a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake +++ b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ # ################################################################################ -# hardcoded tensile version; also in Tensile/Common/Common.py +# hardcoded tensile version; also in Tensile/__init__.py set(TENSILE_VERSION_MAJOR 4) set(TENSILE_VERSION_MINOR 33) set(TENSILE_VERSION_PATCH 0) diff --git a/tensilelite/tox.ini b/tensilelite/tox.ini index 81d82e6384..5b7daee4ed 100644 --- a/tensilelite/tox.ini +++ b/tensilelite/tox.ini @@ -1,5 +1,7 @@ [tox] envlist = py35,py36,py27,lint +labels = + static = format, isort [testenv] # Some versions of Pytest versions have a bug: @@ -25,6 +27,30 @@ deps = flake8 commands = flake8 {toxinidir}/Tensile +[testenv:format] +description = "Formats code so developers don't have to" +skip_install = true +deps = black==24.4 +commands = + black \ + --line-length=100 \ + {toxinidir}/Tensile/Common \ + {toxinidir}/Tensile/TensileCreateLibrary \ + {toxinidir}/Tensile/Utilities/Decorators \ + {posargs} + +[testenv:isort] +description = "Sorts import statements for less merge conflicts" +skip_install = true +deps = isort==5.13.2 +commands = + isort \ + --profile=black \ + {toxinidir}/Tensile/Common \ + {toxinidir}/Tensile/TensileCreateLibrary \ + {toxinidir}/Tensile/Utilities/Decorators \ + {posargs} + [testenv:pre_commit] basepython = python3 deps = @@ -39,9 +65,6 @@ commands = [flake8] exclude = .git,build*,dist,.cache,*~ max-line-length = 132 -ignore = - # All E class violations are errors reported by pycodestyle - E, - # All W class violations are warnings reported by pycodestyle - W - # This mostly just leaves pyflakes errors. +# E/W class violations are errors/warnings reported by pycodestyle, respectively +# This mostly just leaves pyflake errors. +ignore = E, W From 92c13f370e339a0cdf4ddd793c064460b6f15213 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:29:02 -0600 Subject: [PATCH 028/105] chore: move global dependenct funcs to GlobalParameters.py --- tensilelite/Tensile/ClientExecutable.py | 8 +- tensilelite/Tensile/ClientWriter.py | 2 +- .../Tensile/Common/GlobalParameters.py | 359 ++++++++++++++- tensilelite/Tensile/Common/Utilities.py | 425 ++---------------- .../Tensile/TensileCreateLibrary/Run.py | 2 +- 5 files changed, 379 insertions(+), 417 deletions(-) diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index ec5934dc63..ecf972b252 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -44,13 +44,13 @@ def generate(self): args += [self.sourceDir] print2(' '.join(args)) - with ClientExecutionLock(): + with ClientExecutionLock(globalParameters["ClientExecutionLockPath"]): subprocess.check_call(args, cwd=ensurePath(self.buildDir)) def build(self): args = ['make', '-j'] print2(' '.join(args)) - with ClientExecutionLock(): + with ClientExecutionLock(globalParameters["ClientExecutionLockPath"]): subprocess.check_call(args, cwd=self.buildDir) def builtPath(self, path, *paths): @@ -58,7 +58,7 @@ def builtPath(self, path, *paths): def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cCompiler: str): sourcedir = SOURCE_PATH - + builddir = ensurePath(builddir) options = {'CMAKE_BUILD_TYPE': globalParameters["CMakeBuildType"], diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 3811861124..5f0e5fc98a 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -194,7 +194,7 @@ def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: buildPath = ensurePath(outputPath / "build") runScriptName = writeRunScript(buildPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, buildPath, configPaths) - with ClientExecutionLock(): + with ClientExecutionLock(globalParameters["ClientExecutionLockPath"]): process = subprocess.Popen(runScriptName, cwd=buildPath) process.communicate() diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 6556bbe377..7bdd695ef4 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -24,27 +24,21 @@ import math import os.path +import subprocess +import sys import time from collections import OrderedDict from copy import deepcopy from Tensile import __version__ -startTime = time.time() +from .Architectures import gfxToIsa, isaToGfx +from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps +from .Utilities import locateExe, versionIsCompatible -# print level -# 0 - user wants no printing -# 1 - user wants limited prints -# 2 - user wants full prints +startTime = time.time() -################################################################################ -# Global Parameters -################################################################################ globalParameters = OrderedDict() - -######################################## -# common -######################################## globalParameters["MinimumRequiredVersion"] = ( "0.0.0" # which version of tensile is required to handle all the features required by this configuration file ) @@ -1514,3 +1508,344 @@ "LibraryType": "GridBased", "SolutionImportanceMin": 0.01, # = 0.01=1% total time saved by keeping this solution } + + +################################################################################ +# Printing +# 0 - user wants no printing +# 1 - user wants limited prints +# 2 - user wants full prints +################################################################################ +def print1(message): + if globalParameters["PrintLevel"] >= 1: + print(message) + sys.stdout.flush() + + +def print2(message): + if globalParameters["PrintLevel"] >= 2: + print(message) + sys.stdout.flush() + + +def printWarning(message): + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() + + +def printExit(message): + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) + + +################################################################################ +# Is query version compatible with current version +# a yaml file is compatible with tensile if +# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step +################################################################################ +def detectGlobalCurrentISA_(detectionTool): + """ + Returns returncode if detection failure + """ + global globalParameters + + if globalParameters["CurrentISA"] == (0, 0, 0) and detectionTool: + process = subprocess.run([detectionTool], stdout=subprocess.PIPE) + if os.name == "nt": + line = "" + for line_in in process.stdout.decode().splitlines(): + if "gcnArchName" in line_in: + line += line_in.split()[1] + break # detemine if hipinfo will support multiple arch + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + isaToGfx(arch)) + globalParameters["CurrentISA"] = arch + else: + archList = [] + for line in process.stdout.decode().split("\n"): + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in globalParameters["SupportedISA"]: + print1("# Detected local GPU with ISA: " + isaToGfx(arch)) + archList.append(arch) + if len(archList) > 0: + globalParameters["CurrentISA"] = archList[globalParameters["Device"]] + if process.returncode: + printWarning("%s exited with code %u" % (detectionTool, process.returncode)) + return process.returncode + return 0 + + +def detectGlobalCurrentISA(): + """ + Returns returncode if detection failure + """ + errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) + if errorCode: + printWarning("Attempting to detect ISA with rocm_agent_enumerator") + return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) + return errorCode + + +def restoreDefaultGlobalParameters(): + """ + Restores `globalParameters` back to defaults. + """ + global globalParameters + global defaultGlobalParameters + # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would + # result in dangling references, specifically in Tensile.Tensile(). + globalParameters.clear() + for key, value in deepcopy(defaultGlobalParameters).items(): + globalParameters[key] = value + + +def printTable(rows): + rows = list([[str(cell) for cell in row] for row in rows]) + colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) + + for row in rows: + for width, cell in zip(colWidths, row): + pad = " " * (width - len(cell)) + print(pad, cell, sep="", end=" ") + print() + + +def printCapTable(parameters): + import itertools + + archs = [(0, 0, 0)] + parameters["SupportedISA"] + gfxNames = list(map(isaToGfx, archs)) + + headerRow = ["cap"] + gfxNames + + def capRow(caps, cap): + return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] + + allAsmCaps = set( + itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) + ) + allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) + asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + + allArchCaps = set( + itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) + ) + allArchCaps = sorted(allArchCaps) + archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + + printTable([headerRow] + asmCapRows + archCapRows) + + +def assignGlobalParameters(config, cxxCompiler=None): + """ + Assign Global Parameters + Each global parameter has a default parameter, and the user + can override them, those overridings happen here + """ + + global globalParameters + + # Minimum Required Version + if "MinimumRequiredVersion" in config: + if not versionIsCompatible(config["MinimumRequiredVersion"]): + printExit( + "Config file requires version=%s is not compatible with current Tensile version=%s" + % (config["MinimumRequiredVersion"], __version__) + ) + + # User-specified global parameters + print2("GlobalParameters:") + for key in globalParameters: + defaultValue = globalParameters[key] + if key in config: + configValue = config[key] + if configValue == defaultValue: + print2(" %24s: %8s (same)" % (key, configValue)) + else: + print2(" %24s: %8s (overriden)" % (key, configValue)) + else: + print2(" %24s: %8s (unspecified)" % (key, defaultValue)) + + globalParameters["ROCmPath"] = "/opt/rocm" + if "ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") + if "TENSILE_ROCM_PATH" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") + if os.name == "nt" and "HIP_DIR" in os.environ: + globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM + globalParameters["CmakeCxxCompiler"] = None + if "CMAKE_CXX_COMPILER" in os.environ: + globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") + if "CMAKE_C_COMPILER" in os.environ: + globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") + + globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") + + # ROCm AMD GPU Arch Path + # ROCm Agent Enumerator Path + if os.name == "nt": + globalParameters["AMDGPUArchPath"] = locateExe( + globalParameters["ROCmBinPath"], "hipinfo.exe" + ) + globalParameters["ROCmAgentEnumeratorPath"] = locateExe( + globalParameters["ROCmBinPath"], "hipinfo.exe" + ) + else: + globalParameters["AMDGPUArchPath"] = locateExe( + globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch" + ) + globalParameters["ROCmAgentEnumeratorPath"] = locateExe( + globalParameters["ROCmBinPath"], "rocm_agent_enumerator" + ) + + globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") + globalParameters["ROCmLdPath"] = locateExe( + os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld" + ) + + globalParameters["ExtractKernelPath"] = locateExe( + os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel" + ) + + if "AMDGPUArchPath" in config: + globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] + + if "AsanBuild" in config: + globalParameters["AsanBuild"] = config["AsanBuild"] + + if "KeepBuildTmp" in config: + globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] + + if "CodeObjectVersion" in config: + globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] + + # read current gfx version + returncode = detectGlobalCurrentISA() + if globalParameters["CurrentISA"] == (0, 0, 0): + printWarning( + "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." + % globalParameters["SupportedISA"] + ) + if returncode: + if os.name == "nt": + globalParameters["CurrentISA"] = (9, 0, 6) + printWarning("Failed to detect ISA so forcing (gfx906) on windows") + + globalParameters["AsmCaps"] = {} + globalParameters["ArchCaps"] = {} + globalParameters["AsmBugs"] = {} + + for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: + globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) + globalParameters["ArchCaps"][v] = initArchCaps(v) + globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + + if globalParameters["PrintLevel"] >= 1: + printCapTable(globalParameters) + + globalParameters["SupportedISA"] = list( + [ + i + for i in globalParameters["SupportedISA"] + if globalParameters["AsmCaps"][i]["SupportedISA"] + ] + ) + + validParameters["ISA"] = [(0, 0, 0), *globalParameters["SupportedISA"]] + + # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future + # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of + # '0.0.0' will persist + + # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. + # The alternative would be to install the `distro` package. + # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution + + # The following try except block computes the hipcc version + # TODO: hipcc is deprecated, this block should be removed. + try: + compiler = "hipcc" + output = subprocess.run( + [compiler, "--version"], check=True, stdout=subprocess.PIPE + ).stdout.decode() + + for line in output.split("\n"): + if "HIP version" in line: + globalParameters["HipClangVersion"] = line.split()[2] + print1("# Found hipcc version " + globalParameters["HipClangVersion"]) + + except (subprocess.CalledProcessError, OSError) as e: + printWarning("Error: {} running {} {} ".format("hipcc", "--version", e)) + + # The following keys may be present in the config, but are not (or no longer) global parameters. + ignoreKeys = [ + "UseCompression", + "CxxCompiler", + "CCompiler", + "OffloadBundler", + "Assembler", + "LogicPath", + "LogicFilter", + "OutputPath", + "Experimental", + "GenSolTable", + ] + for key in config: + if key in ignoreKeys: + continue + value = config[key] + if key not in globalParameters: + printWarning("Global parameter %s = %s unrecognised." % (key, value)) + globalParameters[key] = value + + +def setupRestoreClocks(): + import atexit + + def restoreClocks(): + if globalParameters["PinClocks"]: + rsmi = globalParameters["ROCmSMIPath"] + subprocess.call([rsmi, "-d", "0", "--resetclocks"]) + subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) + + atexit.register(restoreClocks) + + +setupRestoreClocks() + + +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) + + +def checkParametersAreValid(param, validParams): + """Ensures paramaters in params exist and have valid values as specified by validParames""" + (name, values) = param + if name == "ProblemSizes": + return + elif name == "InternalSupportParams": + return + + if name not in validParams: + printExit( + "Invalid parameter name: {}\nValid parameters are {}.".format( + name, sorted(validParameters.keys()) + ) + ) + + for value in values: + if validParams[name] != -1 and value not in validParams[name]: + msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." + msgExt = ( + " (only first 32 combos printed)\nRefer to Common.py for more info" + if len(validParams[name]) > 32 + else "" + ) + printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 3b746a6f94..291ad52c28 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -2,22 +2,14 @@ import math import os import re -import subprocess import sys import time -import warnings -from copy import deepcopy from enum import Enum -from typing import Optional +from typing import List, Tuple -from .Architectures import gfxToIsa, isaToGfx -from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps -from .GlobalParameters import ( - __version__, - defaultGlobalParameters, - globalParameters, - validParameters, -) +from Tensile import __version__ + +from .Architectures import isaToGfx # get param values from structures. @@ -31,39 +23,8 @@ def hasParam(name, structure): return name in structure else: return name == structure - # printExit("structure %s is not list or dict" % structure) - - -################################################################################ -# Print Debug -################################################################################ -def print1(message): - if globalParameters["PrintLevel"] >= 1: - print(message) - sys.stdout.flush() - - -def print2(message): - if globalParameters["PrintLevel"] >= 2: - print(message) - sys.stdout.flush() - -def printWarning(message): - print("Tensile::WARNING: %s" % message) - sys.stdout.flush() - -def printExit(message): - print("Tensile::FATAL: %s" % message) - sys.stdout.flush() - sys.exit(-1) - - -################################################################################ -# Locate Executables -# rocm-smi, hip-clang, rocm_agent_enumerator, clang-offload-bundler -################################################################################ def isExe(filePath): return os.path.isfile(filePath) and os.access(filePath, os.X_OK) @@ -81,46 +42,39 @@ def locateExe(defaultPath, exeName): # /opt/rocm/bin, hip-clang return None -def which(p): - if "CMAKE_CXX_COMPILER" in os.environ and os.path.isfile(os.environ["CMAKE_CXX_COMPILER"]): - return os.environ["CMAKE_CXX_COMPILER"] - if os.name == "nt": - exes = [ - p + x for x in [".exe", "", ".bat"] - ] # bat may be front end for file with no extension - else: - exes = [p + x for x in ["", ".exe", ".bat"]] - system_path = os.environ["PATH"].split(os.pathsep) - for dirname in system_path + [globalParameters["ROCmBinPath"]]: - for exe in exes: - candidate = os.path.join(os.path.expanduser(dirname), exe) - if os.path.isfile(candidate): - return candidate - return None +def splitArchs(params: dict, fromTensile=False) -> Tuple[List[str], List[str]]: + """ + Splits and processes the architecture strings based on the provided parameters. + Args: + params: A dictionary of global parameters. + fromTensile: A flag indicating if the function is called from the context of Tensile. + + Returns: + A tuple containing two lists: + - archs: A list of architecture strings with ``-`` instead of ``:`` + - cmdlineArchs: A list of architecture strings that retain ``:`` characters. + """ -def splitArchs(fromTensile=False): - # Helper for architecture def isSupported(arch): return ( - globalParameters["AsmCaps"][arch]["SupportedISA"] - and globalParameters["AsmCaps"][arch]["SupportedSource"] + params["AsmCaps"][arch]["SupportedISA"] and params["AsmCaps"][arch]["SupportedSource"] ) - if ";" in globalParameters["Architecture"]: - wantedArchs = globalParameters["Architecture"].split(";") + if ";" in params["Architecture"]: + wantedArchs = params["Architecture"].split(";") else: - wantedArchs = globalParameters["Architecture"].split("_") + wantedArchs = params["Architecture"].split("_") archs = [] cmdlineArchs = [] if "all" in wantedArchs: - for arch in globalParameters["SupportedISA"]: + for arch in params["SupportedISA"]: if isSupported(arch): if arch in [(9, 0, 6), (9, 0, 8), (9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)]: if arch == (9, 0, 10): archs += [isaToGfx(arch) + "-xnack+"] cmdlineArchs += [isaToGfx(arch) + ":xnack+"] - if globalParameters["AsanBuild"]: + if params["AsanBuild"]: archs += [isaToGfx(arch) + "-xnack+"] cmdlineArchs += [isaToGfx(arch) + ":xnack+"] else: @@ -136,7 +90,7 @@ def isSupported(arch): # if calling from the context of Tensile we only want the arch associated with the current ISA if fromTensile: - gfx = isaToGfx(globalParameters["CurrentISA"]) + gfx = isaToGfx(params["CurrentISA"]) archs = set(a for a in archs if gfx in a) cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) @@ -179,32 +133,6 @@ def versionIsCompatible(queryVersionString): return True -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit( - "Invalid parameter name: {}\nValid parameters are {}.".format( - name, sorted(validParameters.keys()) - ) - ) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = ( - " (only first 32 combos printed)\nRefer to Common.py for more info" - if len(validParams[name]) > 32 - else "" - ) - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) - - ################################################################################ # Progress Bar Printing # prints "||||" up to width @@ -353,311 +281,10 @@ def hash_objs(*objs, **kwargs): return hash(tuple(objs)) -def showwarning(message, category, filename, lineno, file=None, line=None): - msg = f"> {category.__name__}: {message}" - print(msg) - - -warnings.showwarning = showwarning - - -################################################################################ -# Is query version compatible with current version -# a yaml file is compatible with tensile if -# tensile.major == yaml.major and tensile.minor.step > yaml.minor.step -################################################################################ -def detectGlobalCurrentISA_(detectionTool): - """ - Returns returncode if detection failure - """ - global globalParameters - - if globalParameters["CurrentISA"] == (0, 0, 0) and detectionTool: - process = subprocess.run([detectionTool], stdout=subprocess.PIPE) - if os.name == "nt": - line = "" - for line_in in process.stdout.decode().splitlines(): - if "gcnArchName" in line_in: - line += line_in.split()[1] - break # detemine if hipinfo will support multiple arch - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - globalParameters["CurrentISA"] = arch - else: - archList = [] - for line in process.stdout.decode().split("\n"): - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - archList.append(arch) - if len(archList) > 0: - globalParameters["CurrentISA"] = archList[globalParameters["Device"]] - if process.returncode: - printWarning("%s exited with code %u" % (detectionTool, process.returncode)) - return process.returncode - return 0 - - -def detectGlobalCurrentISA(): - """ - Returns returncode if detection failure - """ - errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) - if errorCode: - printWarning("Attempting to detect ISA with rocm_agent_enumerator") - return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) - return errorCode - - -def restoreDefaultGlobalParameters(): - """ - Restores `globalParameters` back to defaults. - """ - global globalParameters - global defaultGlobalParameters - # Can't just assign globalParameters = deepcopy(defaultGlobalParameters) because that would - # result in dangling references, specifically in Tensile.Tensile(). - globalParameters.clear() - for key, value in deepcopy(defaultGlobalParameters).items(): - globalParameters[key] = value - - -def printTable(rows): - rows = list([[str(cell) for cell in row] for row in rows]) - colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - - for row in rows: - for width, cell in zip(colWidths, row): - pad = " " * (width - len(cell)) - print(pad, cell, sep="", end=" ") - print() - - -def printCapTable(parameters): - import itertools - - archs = [(0, 0, 0)] + parameters["SupportedISA"] - gfxNames = list(map(isaToGfx, archs)) - - headerRow = ["cap"] + gfxNames - - def capRow(caps, cap): - return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] - - allAsmCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) - ) - allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) - asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] - - allArchCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) - ) - allArchCaps = sorted(allArchCaps) - archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] - - printTable([headerRow] + asmCapRows + archCapRows) - - -def assignGlobalParameters(config, cxxCompiler=None): - """ - Assign Global Parameters - Each global parameter has a default parameter, and the user - can override them, those overridings happen here - """ - - global globalParameters - - # Minimum Required Version - if "MinimumRequiredVersion" in config: - if not versionIsCompatible(config["MinimumRequiredVersion"]): - printExit( - "Config file requires version=%s is not compatible with current Tensile version=%s" - % (config["MinimumRequiredVersion"], __version__) - ) - - # User-specified global parameters - print2("GlobalParameters:") - for key in globalParameters: - defaultValue = globalParameters[key] - if key in config: - configValue = config[key] - if configValue == defaultValue: - print2(" %24s: %8s (same)" % (key, configValue)) - else: - print2(" %24s: %8s (overriden)" % (key, configValue)) - else: - print2(" %24s: %8s (unspecified)" % (key, defaultValue)) - - globalParameters["ROCmPath"] = "/opt/rocm" - if "ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("ROCM_PATH") - if "TENSILE_ROCM_PATH" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("TENSILE_ROCM_PATH") - if os.name == "nt" and "HIP_DIR" in os.environ: - globalParameters["ROCmPath"] = os.environ.get("HIP_DIR") # windows has no ROCM - globalParameters["CmakeCxxCompiler"] = None - if "CMAKE_CXX_COMPILER" in os.environ: - globalParameters["CmakeCxxCompiler"] = os.environ.get("CMAKE_CXX_COMPILER") - if "CMAKE_C_COMPILER" in os.environ: - globalParameters["CmakeCCompiler"] = os.environ.get("CMAKE_C_COMPILER") - - globalParameters["ROCmBinPath"] = os.path.join(globalParameters["ROCmPath"], "bin") - - # ROCm AMD GPU Arch Path - # ROCm Agent Enumerator Path - if os.name == "nt": - globalParameters["AMDGPUArchPath"] = locateExe( - globalParameters["ROCmBinPath"], "hipinfo.exe" - ) - globalParameters["ROCmAgentEnumeratorPath"] = locateExe( - globalParameters["ROCmBinPath"], "hipinfo.exe" - ) - else: - globalParameters["AMDGPUArchPath"] = locateExe( - globalParameters["ROCmPath"], "llvm/bin/amdgpu-arch" - ) - globalParameters["ROCmAgentEnumeratorPath"] = locateExe( - globalParameters["ROCmBinPath"], "rocm_agent_enumerator" - ) - - globalParameters["ROCmSMIPath"] = locateExe(globalParameters["ROCmBinPath"], "rocm-smi") - globalParameters["ROCmLdPath"] = locateExe( - os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld" - ) - - globalParameters["ExtractKernelPath"] = locateExe( - os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel" - ) - - if "AMDGPUArchPath" in config: - globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] - - if "AsanBuild" in config: - globalParameters["AsanBuild"] = config["AsanBuild"] - - if "KeepBuildTmp" in config: - globalParameters["KeepBuildTmp"] = config["KeepBuildTmp"] - - if "CodeObjectVersion" in config: - globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - - # read current gfx version - returncode = detectGlobalCurrentISA() - if globalParameters["CurrentISA"] == (0, 0, 0): - printWarning( - "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." - % globalParameters["SupportedISA"] - ) - if returncode: - if os.name == "nt": - globalParameters["CurrentISA"] = (9, 0, 6) - printWarning("Failed to detect ISA so forcing (gfx906) on windows") - - globalParameters["AsmCaps"] = {} - globalParameters["ArchCaps"] = {} - globalParameters["AsmBugs"] = {} - - for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: - globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) - globalParameters["ArchCaps"][v] = initArchCaps(v) - globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) - - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) - - globalParameters["SupportedISA"] = list( - [ - i - for i in globalParameters["SupportedISA"] - if globalParameters["AsmCaps"][i]["SupportedISA"] - ] - ) - - validParameters["ISA"] = [(0, 0, 0), *globalParameters["SupportedISA"]] - - # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future - # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of - # '0.0.0' will persist - - # Due to platform.linux_distribution() being deprecated, just try to run dpkg regardless. - # The alternative would be to install the `distro` package. - # See https://docs.python.org/3.7/library/platform.html#platform.linux_distribution - - # The following try except block computes the hipcc version - try: - if os.name == "nt": - compileArgs = ["perl"] + [which("hipcc")] + ["--version"] - output = subprocess.run(compileArgs, check=True, stdout=subprocess.PIPE).stdout.decode() - else: - compiler = "hipcc" - output = subprocess.run( - [compiler, "--version"], check=True, stdout=subprocess.PIPE - ).stdout.decode() - - for line in output.split("\n"): - if "HIP version" in line: - globalParameters["HipClangVersion"] = line.split()[2] - print1("# Found hipcc version " + globalParameters["HipClangVersion"]) - - except (subprocess.CalledProcessError, OSError) as e: - printWarning("Error: {} running {} {} ".format("hipcc", "--version", e)) - - # The following keys may be present in the config, but are not (or no longer) global parameters. - ignoreKeys = [ - "UseCompression", - "CxxCompiler", - "CCompiler", - "OffloadBundler", - "Assembler", - "LogicPath", - "LogicFilter", - "OutputPath", - "Experimental", - "GenSolTable", - ] - for key in config: - if key in ignoreKeys: - continue - value = config[key] - if key not in globalParameters: - printWarning("Global parameter %s = %s unrecognised." % (key, value)) - globalParameters[key] = value - - -def setupRestoreClocks(): - import atexit - - def restoreClocks(): - if globalParameters["PinClocks"]: - rsmi = globalParameters["ROCmSMIPath"] - subprocess.call([rsmi, "-d", "0", "--resetclocks"]) - subprocess.call([rsmi, "-d", "0", "--setfan", "50"]) - - atexit.register(restoreClocks) - - -setupRestoreClocks() - - -################################################################################ -# Assign Parameters -# populate dst with src[key] else give it the default/backup value -################################################################################ -def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): - if key in sourceDictionary: - destinationDictionary[key] = deepcopy(sourceDictionary[key]) - else: - destinationDictionary[key] = deepcopy(defaultDictionary[key]) - - -def ClientExecutionLock(): - if not globalParameters["ClientExecutionLockPath"]: +def ClientExecutionLock(lockPath: str): + if not lockPath: return open(os.devnull) import filelock - return filelock.FileLock(globalParameters["ClientExecutionLockPath"]) + return filelock.FileLock(lockPath) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 6bbc56b1e2..445037c653 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -41,8 +41,8 @@ architectureMap, assignGlobalParameters, ensurePath, - isaToGfx, globalParameters, + isaToGfx, print1, print2, printExit, From 76d257907a554135877330f471d3d815cb7c74ba Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:29:02 -0600 Subject: [PATCH 029/105] fix: pass global params to splitArchs --- tensilelite/Tensile/Toolchain/Source.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 24147869a8..59ee225071 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -165,11 +165,11 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa def buildSourceCodeObjectFiles( - toolchain: SourceToolchain, - destDir: Union[Path, str], - tmpObjDir: Union[Path, str], - includeDir: Union[Path, str], - kernelPath: Union[Path, str], + toolchain: SourceToolchain, + destDir: Union[Path, str], + tmpObjDir: Union[Path, str], + includeDir: Union[Path, str], + kernelPath: Union[Path, str], fromTensile: bool ) -> List[str]: """Compiles a HIP source code file into a code object file. @@ -197,7 +197,7 @@ def buildSourceCodeObjectFiles( coPathsRaw = [] coPaths= [] - _, cmdlineArchs = splitArchs(fromTensile) + _, cmdlineArchs = splitArchs(globalParameters, fromTensile) objPath = str(tmpObjDir / objFilename) toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) From 57cb9afc27725888078de0a2a6aba132c3463dac Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 7 Feb 2025 16:43:51 -0600 Subject: [PATCH 030/105] style: don't format caps funcs --- tensilelite/Tensile/Common/Capabilities.py | 293 ++++++--------------- 1 file changed, 76 insertions(+), 217 deletions(-) diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index 5f0977abbd..0028b3c69b 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -2,7 +2,7 @@ from functools import lru_cache from typing import Tuple -from .Architectures import isaToGfx +from .Architectures import getGfxName def _tryAssembler( @@ -27,7 +27,7 @@ def _tryAssembler( "assembler", "-target", "amdgcn-amdhsa", - "-mcpu=" + isaToGfx(isaVersion), + "-mcpu=" + getGfxName(isaVersion), *options, "-", ] @@ -58,213 +58,77 @@ def _tryAssembler( def initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: """Determine assembler capabilities by testing short instructions sequences""" rv = {} - rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) - rv["HasExplicitCO"] = _tryAssembler( - isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug - ) - rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) - - rv["HasDirectToLds"] = _tryAssembler( - isaVersion, - assemblerPath, - "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", - isDebug, - ) or _tryAssembler( - isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug - ) - rv["HasAddLshl"] = _tryAssembler( - isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug - ) - rv["HasLshlOr"] = _tryAssembler( - isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug - ) - rv["HasSMulHi"] = _tryAssembler( - isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug - ) + # fmt: off + rv["SupportedISA"] = _tryAssembler(isaVersion, assemblerPath, "", isDebug) + rv["HasExplicitCO"] = _tryAssembler(isaVersion, assemblerPath, "v_add_co_u32 v0,vcc,v0,1", isDebug) + rv["HasExplicitNC"] = _tryAssembler(isaVersion, assemblerPath, "v_add_nc_u32 v0,v0,1", isDebug) - rv["HasMFMA_explictB"] = _tryAssembler( - isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug - ) - rv["HasMFMA"] = ( - _tryAssembler( - isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug - ) - or rv["HasMFMA_explictB"] - ) - rv["HasMFMA_f64"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", - isDebug, - ) or _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", - isDebug, - ) - rv["HasMFMA_bf16_1k"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", - isDebug, - ) - rv["HasMFMA_f8"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", - isDebug, - ) - rv["HasMFMA_b8"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", - isDebug, - ) + rv["HasDirectToLds"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v36, s[24:27], s28 offen offset:0 lds", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v36, s[24:27], s28 offen offset:0 lds", isDebug) + rv["HasAddLshl"] = _tryAssembler(isaVersion, assemblerPath, "v_add_lshl_u32 v47, v36, v34, 0x2", isDebug) + rv["HasLshlOr"] = _tryAssembler(isaVersion, assemblerPath, "v_lshl_or_b32 v47, v36, 0x2, v34", isDebug) + rv["HasSMulHi"] = _tryAssembler(isaVersion, assemblerPath, "s_mul_hi_u32 s47, s36, s34", isDebug) - rv["HasMFMA_xf32"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", - isDebug, - ) - rv["HasSMFMA"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", - isDebug, - ) - rv["HasWMMA"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", - isDebug, - ) or _tryAssembler( - isaVersion, - assemblerPath, - "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", - isDebug, - ) - rv["HasWMMA_V1"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", - isDebug, - ) - rv["HasWMMA_V2"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", - isDebug, - ) + rv["HasMFMA_explictB"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]", isDebug) + rv["HasMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x2bf16 a[0:31], v32, v33, a[0:31]", isDebug) or rv["HasMFMA_explictB"] + rv["HasMFMA_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) or _tryAssembler(isaVersion, assemblerPath, "v_mfma_f64_16x16x4_f64 v[0:7], v[32:33], v[36:37], v[0:7]", isDebug) + rv["HasMFMA_bf16_1k"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4bf16_1k a[0:31], v[32:33], v[36:37], a[0:31]", isDebug) + rv["HasMFMA_f8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_fp8_fp8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) + rv["HasMFMA_b8"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_16x16x32_bf8_bf8 a[0:3], v[2:3], v[4:5], a[0:3]", isDebug) - rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) + rv["HasMFMA_xf32"] = _tryAssembler(isaVersion, assemblerPath, "v_mfma_f32_32x32x4_xf32 a[0:15], v[32:33], v[36:37], a[0:15]", isDebug) + rv["HasSMFMA"] = _tryAssembler(isaVersion, assemblerPath, "v_smfmac_f32_32x32x16_f16 a[0:15], v[32:33], v[36:39], v[40]", isDebug) + rv["HasWMMA"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) + rv["HasWMMA_V1"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:15], v[16:23], v[0:3]", isDebug) + rv["HasWMMA_V2"] = _tryAssembler(isaVersion, assemblerPath, "v_wmma_f32_16x16x16_f16 v[0:3], v[8:9], v[16:17], v[0:3]", isDebug) - rv["v_fma_f16"] = _tryAssembler( - isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug - ) - rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) + rv["v_mac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f16 v47, v36, v34", isDebug) - rv["v_pk_fma_f16"] = _tryAssembler( - isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug - ) - rv["v_pk_fmac_f16"] = _tryAssembler( - isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug - ) + rv["v_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0,0]", isDebug) + rv["v_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f16 v47, v36, v34", isDebug) - rv["v_pk_add_f32"] = _tryAssembler( - isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug - ) - rv["v_pk_mul_f32"] = _tryAssembler( - isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug - ) + rv["v_pk_fma_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34, v47, op_sel:[0,0,0]", isDebug) + rv["v_pk_fmac_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_fma_f16 v47, v36, v34", isDebug) - rv["v_pk_mul_f32"] = _tryAssembler( - isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug - ) + rv["v_pk_add_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_add_f32 v[48:49], v[36:37], v[0:1]", isDebug) + rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[48:49], v[36:37], v[0:1]", isDebug) - rv["v_mad_mix_f32"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", - isDebug, - ) - rv["v_fma_mix_f32"] = _tryAssembler( - isaVersion, - assemblerPath, - "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", - isDebug, - ) + rv["v_pk_mul_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_pk_mul_f32 v[20:21], v[18:19], v[20:21]", isDebug) - rv["v_dot2_f32_f16"] = _tryAssembler( - isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug - ) - rv["v_dot2c_f32_f16"] = _tryAssembler( - isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug - ) or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) + rv["v_mad_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mad_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) + rv["v_fma_mix_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_mix_f32 v47, v36, v34, v47, op_sel:[0,0,0] op_sel_hi:[1,1,0]", isDebug) - rv["v_dot4_i32_i8"] = _tryAssembler( - isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug - ) - rv["v_dot4c_i32_i8"] = _tryAssembler( - isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug - ) - rv["VOP3v_dot4_i32_i8"] = _tryAssembler( - isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug - ) + rv["v_dot2_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2_f32_f16 v20, v36, v34, v20", isDebug) + rv["v_dot2c_f32_f16"] = _tryAssembler(isaVersion, assemblerPath, "v_dot2c_f32_f16 v47, v36, v34", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "v_dot2acc_f32_f16 v47, v36, v34", isDebug) - rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) - rv["v_fma_f32"] = _tryAssembler( - isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug - ) - rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) + rv["v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34", isDebug) + rv["v_dot4c_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4c_i32_i8 v47, v36, v34", isDebug) + rv["VOP3v_dot4_i32_i8"] = _tryAssembler(isaVersion, assemblerPath, "v_dot4_i32_i8 v47, v36, v34, v47", isDebug) - rv["v_fma_f64"] = _tryAssembler( - isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug - ) + rv["v_mac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_mac_f32 v20, v21, v22", isDebug) + rv["v_fma_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f32 v20, v21, v22, v23", isDebug) + rv["v_fmac_f32"] = _tryAssembler(isaVersion, assemblerPath, "v_fmac_f32 v20, v21, v22", isDebug) - rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) + rv["v_fma_f64"] = _tryAssembler(isaVersion, assemblerPath, "v_fma_f64 v[20:21], v[22:23], v[24:25], v[20:21]", isDebug) - rv["HasAtomicAdd"] = _tryAssembler( - isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug - ) or _tryAssembler( - isaVersion, - assemblerPath, - "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", - isDebug, - ) - rv["HasGLCModifier"] = _tryAssembler( - isaVersion, - assemblerPath, - "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", - isDebug, - ) or _tryAssembler( - isaVersion, - assemblerPath, - "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", - isDebug, - ) - rv["HasMUBUFConst"] = _tryAssembler( - isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug - ) or _tryAssembler( - isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug - ) - rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) + rv["v_mov_b64"] = _tryAssembler(isaVersion, assemblerPath, "v_mov_b64 v[0:1], v[2:3]", isDebug) - rv["HasGLCModifier"] = _tryAssembler( - isaVersion, - assemblerPath, - "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", - isDebug, - ) + rv["HasAtomicAdd"] = _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:0", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:0", isDebug) + rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], null, offen offset:0, glc", isDebug) + rv["HasMUBUFConst"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dword v40, v36, s[24:27], 1 offen offset:0", isDebug) \ + or _tryAssembler(isaVersion, assemblerPath, "buffer_load_b32 v40, v36, s[24:27], 1 offen offset:0", isDebug) + rv["HasSCMPK"] = _tryAssembler(isaVersion, assemblerPath, "s_cmpk_gt_u32 s56, 0x0", isDebug) - rv["HasNTModifier"] = _tryAssembler( - isaVersion, - assemblerPath, - "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", - isDebug, - ) + rv["HasGLCModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, glc", isDebug) - rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) + rv["HasNTModifier"] = _tryAssembler(isaVersion, assemblerPath, "buffer_load_dwordx4 v[10:13], v[0], s[0:3], 0, offen offset:0, nt", isDebug) + + rv["HasNewBarrier"] = _tryAssembler(isaVersion, assemblerPath, "s_barrier_wait -1", isDebug) + # fmt: on if _tryAssembler(isaVersion, assemblerPath, "s_waitcnt vmcnt(63)", isDebug): rv["MaxVmcnt"] = 63 @@ -284,32 +148,27 @@ def initAsmCaps(isaVersion, assemblerPath, isDebug) -> dict: @lru_cache() def initArchCaps(isaVersion) -> dict: rv = {} - rv["HasEccHalf"] = isaVersion in [ - (9, 0, 6), - (9, 0, 8), - (9, 0, 10), - (9, 4, 0), - (9, 4, 1), - (9, 4, 2), - ] - rv["Waitcnt0Disabled"] = isaVersion in [(9, 0, 8), (9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["SeparateVscnt"] = isaVersion[0] in (10, 11) - rv["SeparateLGKMcnt"] = isaVersion[0] == (12) - rv["SeparateVMcnt"] = isaVersion[0] == (12) - rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) - rv["HasWave32"] = isaVersion[0] in (10, 11, 12) - rv["HasAccCD"] = isaVersion in [(9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["ArchAccUnifiedRegs"] = isaVersion in [(9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["CrosslaneWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["ForceStoreSC1"] = isaVersion in [(9, 4, 0), (9, 4, 1)] - rv["TransOpWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["SDWAWait"] = isaVersion in [(9, 4, 0), (9, 4, 1), (9, 4, 2)] - rv["VgprBank"] = isaVersion[0] in (10, 11, 12) - rv["DSLow16NotPreserve"] = isaVersion[0] == (12) + # fmt: off + rv["HasEccHalf"] = (isaVersion in [(9,0,6), (9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["Waitcnt0Disabled"] = (isaVersion in [(9,0,8), (9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["SeparateVscnt"] = isaVersion[0] in (10, 11) + rv["SeparateLGKMcnt"] = isaVersion[0] == (12) + rv["SeparateVMcnt"] = isaVersion[0] == (12) + rv["CMPXWritesSGPR"] = isaVersion[0] not in (10, 11, 12) + rv["HasWave32"] = isaVersion[0] in (10, 11, 12) + rv["HasAccCD"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["ArchAccUnifiedRegs"] = (isaVersion in [(9,0,10), (9,4,0), (9,4,1), (9,4,2)]) + rv["CrosslaneWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["ForceStoreSC1"] = (isaVersion in [(9,4,0), (9,4,1)]) + rv["TransOpWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) + rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) + rv["DSLow16NotPreserve"] = isaVersion[0] == (12) rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) - rv["NoSDWA"] = isaVersion[0] == (12) - rv["VOP3ByteSel"] = isaVersion[0] == (12) - rv["HasFP8_OCP"] = isaVersion[0] == (12) + rv["NoSDWA"] = isaVersion[0] == (12) + rv["VOP3ByteSel"] = isaVersion[0] == (12) + rv["HasFP8_OCP"] = isaVersion[0] == (12) + # fmt: on return rv From 3d4ea710c8fc15adb6027cb7571ba276985f49d8 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 7 Feb 2025 17:07:53 -0600 Subject: [PATCH 031/105] fix: bad import --- tensilelite/Tensile/Common/Capabilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index 0028b3c69b..db20079d5a 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -2,7 +2,7 @@ from functools import lru_cache from typing import Tuple -from .Architectures import getGfxName +from .Architectures import isaToGfx def _tryAssembler( @@ -27,7 +27,7 @@ def _tryAssembler( "assembler", "-target", "amdgcn-amdhsa", - "-mcpu=" + getGfxName(isaVersion), + "-mcpu=" + isaToGfx(isaVersion), *options, "-", ] From afec06977a1be9b5839011017415f7594761a817 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 10 Feb 2025 14:28:16 -0700 Subject: [PATCH 032/105] Add TensileLogic program to verify matrix instructions (#3) --- tensilelite/Tensile/Common/Parallel.py | 6 +- .../Tensile/TensileLogic/ParseArguments.py | 66 +++++ tensilelite/Tensile/TensileLogic/Run.py | 67 +++++ .../TensileLogic/ValidMatrixInstruction.py | 233 ++++++++++++++++++ tensilelite/Tensile/TensileLogic/__init__.py | 1 + tensilelite/Tensile/bin/TensileLogic | 44 ++++ 6 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 tensilelite/Tensile/TensileLogic/ParseArguments.py create mode 100644 tensilelite/Tensile/TensileLogic/Run.py create mode 100644 tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py create mode 100644 tensilelite/Tensile/TensileLogic/__init__.py create mode 100755 tensilelite/Tensile/bin/TensileLogic diff --git a/tensilelite/Tensile/Common/Parallel.py b/tensilelite/Tensile/Common/Parallel.py index 9cd7396eb9..77adfd26c1 100644 --- a/tensilelite/Tensile/Common/Parallel.py +++ b/tensilelite/Tensile/Common/Parallel.py @@ -195,7 +195,9 @@ def ParallelMapReturnAsGenerator(function, objects, message="", enable=True, mul yield result.result() -def ParallelMap2(function, objects, message="", enable=True, multiArg=True, return_as="list"): +def ParallelMap2( + function, objects, message="", enable=True, multiArg=True, return_as="list", procs=None +): """ Generally equivalent to list(map(function, objects)), possibly executing in parallel. @@ -209,6 +211,8 @@ def ParallelMap2(function, objects, message="", enable=True, multiArg=True, retu from .GlobalParameters import globalParameters + threadCount = procs if procs else CPUThreadCount(enable) + threadCount = CPUThreadCount(enable) if threadCount <= 1 and globalParameters["ShowProgressBar"]: diff --git a/tensilelite/Tensile/TensileLogic/ParseArguments.py b/tensilelite/Tensile/TensileLogic/ParseArguments.py new file mode 100644 index 0000000000..25ce08cc95 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/ParseArguments.py @@ -0,0 +1,66 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from argparse import ArgumentParser +from typing import Any, Dict + +from Tensile.Toolchain.Validators import ToolchainDefaults + + +def parseArguments() -> Dict[str, Any]: + """ + Returns: + A dictionary containing the keys representing options and their values. + """ + + argParser = ArgumentParser( + description="TensileValidateLogic runs critical checks to ensure the " + "integrity of the supplied logic files.", + ) + + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument( + "--check-matrix-instruction", + dest="CheckMatrixInstruction", + action="store_true", + help="Checks that matrix instructions are valid for all target ISAs.", + ) + argParser.add_argument( + "--jobs", + "-j", + dest="Jobs", + action="store", + default=48, + help="Number of worker processes to use during validation checks.", + ) + argParser.add_argument( + "--cxx-compiler", + dest="CxxCompiler", + action="store", + default=ToolchainDefaults.CXX_COMPILER, + help=f"Default: {ToolchainDefaults.CXX_COMPILER}", + ) + args = argParser.parse_args() + + return args diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py new file mode 100644 index 0000000000..a22eab2d22 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -0,0 +1,67 @@ +import yaml +import functools +from pathlib import Path + +from Tensile.Common import globalParameters, assignGlobalParameters, ParallelMap2 +from Tensile.LibraryIO import readYAML +from Tensile.Toolchain.Validators import validateToolchain + +from .ParseArguments import parseArguments +from .ValidMatrixInstruction import validateMatrixInstruction + + +def getParams(cxxCompiler): + gp = globalParameters + + gpcache = Path.cwd() / "gpcache.yaml" + if gpcache.exists(): + with open(gpcache, "r") as f: + gp = yaml.load(f, yaml.CSafeLoader) + else: + assignGlobalParameters({}, cxxCompiler) + with open(gpcache, "w") as f: + yaml.dump(gp, f, yaml.CSafeDumper) + + return gp + + +def runChecks(logicPath, gp, file): + if "Experimental" in file.parts: + return 0, 0 + + keep, total = 0, 0 + solutions = readYAML(file)[5] # Solutions are the 5th index + for s in solutions: + total += 1 + keep += validateMatrixInstruction(s, file.relative_to(logicPath), gp) + print(f">> {file.relative_to(logicPath)}") + return keep, total + + +def main(): + args = parseArguments() + cxxCompiler = validateToolchain(args.CxxCompiler) + gp = getParams(cxxCompiler) + + logicPath = Path(args.LogicPath) + pattern = "**/*.yaml" + files = logicPath.glob(pattern) + print(f"Checking logic files with glob {args.LogicPath}{pattern}") + + if not any([args.CheckMatrixInstruction]): + print("No checks specified. Exiting.") + exit(0) + + fn = functools.partial(runChecks, logicPath, gp) + results = ParallelMap2(fn, files, multiArg=False, procs=args.Jobs) + + keep = sum([x[0] for x in results]) + total = sum([x[1] for x in results]) + + rejects = total - keep + print(f"Total {total} solutions") + print(f"Keep {keep} solutions") + print(f"Reject {rejects} solutions") + + if rejects > 0: + exit(1) diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py new file mode 100644 index 0000000000..7d1ca2c926 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -0,0 +1,233 @@ +import math +from pathlib import Path +from inspect import currentframe, getframeinfo + +MI_KEY: str = "MatrixInstruction" +MI_ENABLED_KEY: str = "EnableMatrixInstruction" + + +validMFMA = {} +validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] +validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] +validMFMA["4xi8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], + [32, 32, 16, 1], + [16, 16, 32, 1], +] +validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] +validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["C"] = validMFMA["S"] +validMFMA["Z"] = validMFMA["D"] +validMFMA["I8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], +] + [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] +validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8"] = validMFMA["F8"] +validMFMA["F8B8"] = validMFMA["F8"] +validMFMA["B8F8"] = validMFMA["F8"] +validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8N"] = validMFMA["F8N"] +validMFMA["F8B8N"] = validMFMA["F8N"] +validMFMA["B8F8N"] = validMFMA["F8N"] +validWMMA = [ + [16, 16, 16, 1], +] +validTT = 32 +validMFMA["_format9"] = [] + +for MFMA in [ + validMFMA["H"], + validMFMA["S"], + validMFMA["B"], + validMFMA["D"], + validMFMA["X"], + validMFMA["F8N"], + validWMMA, +]: + for MI in MFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validMatrixInstructions = ( + [[], [-1]] + + validMFMA["H"] + + validMFMA["S"] + + validMFMA["B"] + + validMFMA["D"] + + validMFMA["B1k"] + + validMFMA["X"] +) +validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] + +validSMFMA = {} +validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["I8"] = validSMFMA["4xi8"] +validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8"] = validSMFMA["F8"] +validSMFMA["F8B8"] = validSMFMA["F8"] +validSMFMA["B8F8"] = validSMFMA["F8"] +validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8N"] = validSMFMA["F8N"] +validSMFMA["F8B8N"] = validSMFMA["F8N"] +validSMFMA["B8F8N"] = validSMFMA["F8N"] +validSMFMA["_format9"] = [] +for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: + for MI in SMFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validSMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] +validMatrixInstructions = ( + validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] +) + + +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" + + +def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): + """ + Validates the matrix instruction configured in the given solution. + + The function performs the following checks: + - Ensures that the solution contains the required keys for matrix instruction support. + - Ensures that the matrix instruction is not empty when it is enabled. + - Validates that the matrix instruction is in the list of valid matrix instructions. + - If the matrix instruction has 9 elements, it performs detailed validation checks: + - Validates the work group dimensions. + - Checks if the matrix instruction is supported by the assembler capabilities (MFMA or WMMA). + - Validates the input per thread for sparse and non-sparse configurations. + - Validates the matrix instruction block, wave group, and wave tile dimensions. + - If the matrix instruction has 4 elements, it ensures that matrix instructions are enabled. + - If the matrix instruction is empty, it ensures that matrix instructions are disabled. + + Args: + solution: The solution to validate. + filepath: The path to the file containing the solution. + params: The global parameters for the solution. + + Raises: + AssertionError: If any of the validation checks fail. + """ + try: + _validateMatrixInstruction(solution, params) + return True + except AssertionError as e: + print(f"Validation failed: {filepath} (index {solution['SolutionIndex']})") + print(f"Error: file: {e}") + return False + + +def _validateMatrixInstruction(solution: dict, params: dict): + """ + Function to validate the matrix instruction for the provided solution. + See exported function for more details. + """ + assert MI_KEY in solution, elineno() + assert MI_ENABLED_KEY in solution, elineno() + assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), elineno() + + isa = tuple(solution["ISA"]) + miFull = solution[MI_KEY] + miEnabled = solution[MI_ENABLED_KEY] + + assert miFull in validMatrixInstructions, elineno() + + if len(solution[MI_KEY]) == 9: + wfsize = solution["WavefrontSize"] + mi = [miFull[0], miFull[1], miFull[2], miFull[3]] + waves = miFull[7] * miFull[8] + miwg0 = miFull[4] * miFull[0] * miFull[7] # Matrix instruction work group 0 + miwg1 = waves * wfsize // miwg0 + + isSparse = solution["ProblemType"]["Sparse"] + miDataType = ( + solution["ProblemType"]["DataType"] + if (not solution["EnableF32XdlMathOp"]) + else solution["ProblemType"]["F32XdlMathOp"] + ) + miBlock = solution["MIBlock"] + miWaveGroup = solution["MIWaveGroup"] + miWaveTile = solution["MIWaveTile"] + miInputPerThread = solution["MIInputPerThread"] + miInputPerThreadA = solution["MIInputPerThreadA"] + miInputPerThreadB = solution["MIInputPerThreadB"] + miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] + + # Check work group + assert solution["WorkGroup"] == [miwg0, miwg1], elineno() + + # Check datatype + if not isSparse: + if params["AsmCaps"][isa]["HasMFMA"]: + if not (miDataType.toChar() in validMFMA and mi in validMFMA[miDataType.toChar()]): + assert miDataType.isBFloat16() and mi in validMFMA["B1k"], elineno() + elif params["AsmCaps"][isa]["HasWMMA"]: + assert mi in validWMMA, elineno() + else: + assert miDataType.toChar() in validSMFMA and mi in validSMFMA[miDataType.toChar()], elineno() + + if (not params["AsmCaps"][isa]["HasMFMA"]) and params["AsmCaps"][isa]["HasWMMA"]: + if isa[0] == 10 or isa[0] == 11: + assert miInputPerThread == mi[2], elineno() + + assert solution["MFMA_BF16_1K"] == False, elineno() + + # Check MIBlock + assert miBlock[0] == mi[0], elineno() + assert miBlock[1] == mi[1], elineno() + assert miBlock[2] == mi[2], elineno() + assert miBlock[3] == mi[3], elineno() + assert miBlock[4] == min(miwg0 // mi[0], mi[3]), elineno() + assert miBlock[5] == mi[3] // miBlock[4], elineno() + + # Check MIWaveGroup + assert miWaveGroup[0] == min((miwg0 // mi[0]) // miBlock[4], waves), elineno() + assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() + + # Check MIWaveTile + assert miWaveTile[0] == mi[5], elineno() + assert miWaveTile[1] == mi[6], elineno() + + # Check MIInputPerThread + assert miInputPerThread == mi[0] * mi[2] * mi[3] // wfsize, elineno() + + # TODO: sparsity in hipBLASLt appears to be unused or always zero + sparseA = not isSparse if isSparse != 2 else False + sparseB = isSparse == 2 if isSparse else False + assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() + assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() + assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() + + assert miEnabled == True, elineno() + elif miFull != [] and len(miFull) == 4: + assert miEnabled == True, elineno() + else: + assert miEnabled == False, elineno() diff --git a/tensilelite/Tensile/TensileLogic/__init__.py b/tensilelite/Tensile/TensileLogic/__init__.py new file mode 100644 index 0000000000..46b158ded5 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/__init__.py @@ -0,0 +1 @@ +from .Run import main diff --git a/tensilelite/Tensile/bin/TensileLogic b/tensilelite/Tensile/bin/TensileLogic new file mode 100755 index 0000000000..d1b70ed72c --- /dev/null +++ b/tensilelite/Tensile/bin/TensileLogic @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +# This script only gets called by CMake + +try: + from Tensile import TensileLogic +except ImportError: + import os.path + import sys + parentdir = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..")) + print(parentdir) + sys.path.append(parentdir) + + from Tensile import TensileLogic + +################################################################################ +# Main +################################################################################ +if __name__ == "__main__": + TensileLogic.main() From 2b4402e74e51599d89a0f6c551f57ea38fe2c001 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:00:37 -0700 Subject: [PATCH 033/105] Remove unused and static global parameters (#5) * fix: purge library-print-debug * feat: make build paths static * feat: remove 'SortProblems' global param * feat: remove 'ExpandRanges' global param * feat: remove 'WavefrontWidth' global param * feat: remove 'ValidateLibrary' global param * feat: remove 'EnableHalf' and 'ClientArgs' global params * feat: prefer profile decorator over 'Profiler' global param * chore: remove 'LibraryPrintDebug' from build_client.yaml * feat: remove 'MaxFileName' from global params --- tensilelite/Tensile/BenchmarkProblems.py | 8 ++--- tensilelite/Tensile/ClientExecutable.py | 4 +-- tensilelite/Tensile/ClientWriter.py | 15 ++++---- tensilelite/Tensile/Common/Constants.py | 10 ++++++ .../Tensile/Common/GlobalParameters.py | 32 ----------------- tensilelite/Tensile/KernelWriter.py | 7 ++-- tensilelite/Tensile/LibraryLogic.py | 34 ++++--------------- tensilelite/Tensile/SolutionStructs.py | 5 +-- .../Tensile/Source/TensileCreateLibrary.cmake | 3 +- tensilelite/Tensile/Tensile.py | 26 ++++---------- tensilelite/Tensile/Tests/build_client.yaml | 1 - tensilelite/Tensile/Tests/conftest.py | 3 +- 12 files changed, 42 insertions(+), 106 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 4d62262340..350ab1fa56 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -42,7 +42,8 @@ from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain from .Common import globalParameters, HR, print1, print2, \ - printExit, printWarning, ensurePath, startTime, tqdm, state + printExit, printWarning, ensurePath, startTime, tqdm, state, \ + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): @@ -386,7 +387,7 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source print(f'No config specified in {globalParameters["ConfigPath"]}, built client only') return - benchmarkDataPath = ensurePath(outputPath / globalParameters["BenchmarkDataPath"]) + benchmarkDataPath = ensurePath(outputPath / BENCHMARK_DATA_DIR) totalTestFails = 0 for benchmarkProblemTypeConfig in config: @@ -399,7 +400,6 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source for idx, sizeGroupConfig in enumerate(problemSizeGroupConfigs): print2("ProblemTypeConfig: {}".format(problemTypeConfig)) problemTypeObj = ProblemType(problemTypeConfig) - globalParameters["EnableHalf"] = problemTypeObj["DataType"].isHalf() # using a suffix to check the csv version (for later addFromCSV()) csvSuffix = "_CSVWinner" if globalParameters["CSVExportWinner"] else "" @@ -416,7 +416,7 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source or not os.path.exists(newResultsFileName): # benchmark problem size group - benchmarkProblemsPath = ensurePath(outputPath / globalParameters["BenchmarkProblemsPath"]) + benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath) totalTestFails += benchmarkErrors diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index ecf972b252..b02a2f0e6a 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -29,7 +29,7 @@ from pathlib import Path from . import SOURCE_PATH -from .Common import globalParameters, print2, ClientExecutionLock, ensurePath +from .Common import globalParameters, print2, ClientExecutionLock, ensurePath, CLIENT_BUILD_DIR class CMakeEnvironment: def __init__(self, sourceDir, buildDir, **options): @@ -81,7 +81,7 @@ def getClientExecutable(cxxCompiler: str, cCompiler: str, builddir): global buildEnv if buildEnv is None: - buildEnv = clientExecutableEnvironment(builddir / globalParameters["ClientBuildPath"], cxxCompiler, cCompiler) + buildEnv = clientExecutableEnvironment(builddir / CLIENT_BUILD_DIR, cxxCompiler, cCompiler) buildEnv.generate() buildEnv.build() diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 5f0e5fc98a..5d3e31367d 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -34,7 +34,8 @@ from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO -from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx +from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, \ + LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex @@ -80,8 +81,8 @@ class ClientLogLevel(Enum): ################################################################################ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): - libraryLogicPath = ensurePath(outputPath / globalParameters["LibraryLogicPath"]) - clientLibraryPath = ensurePath(outputPath / globalParameters["LibraryClientPath"]) + libraryLogicPath = ensurePath(outputPath / LIBRARY_LOGIC_DIR) + clientLibraryPath = ensurePath(outputPath / LIBRARY_CLIENT_DIR) sourcePath = ensurePath(clientLibraryPath / "source") copyStaticFiles(sourcePath) @@ -95,7 +96,6 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): print1("LogicFiles: %s" % logicFiles) functions = [] functionNames = [] - enableHalf = False createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler) subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) @@ -106,8 +106,6 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary) \ = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) - if problemType["DataType"].isHalf(): - enableHalf = True functions.append((scheduleName, problemType)) functionNames.append("tensile_%s" % (problemType)) problemSizes = ProblemSizesMock(exactLogic) if exactLogic else ProblemSizesMockDummy() @@ -154,7 +152,6 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): codeObjectFiles=coList, tileAwareSelection=False, libraryFile=yamlList[0])) - globalParameters["EnableHalf"] = enableHalf forBenchmark = False problemSizes = None @@ -286,7 +283,7 @@ def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cC clientExe = ClientExecutable.getClientExecutable(cxxCompiler, cCompiler, buildDir) for configFile in configPaths: - runScriptFile.write("{} --config-file {} {}\n".format(clientExe, configFile, globalParameters["ClientArgs"])) + runScriptFile.write("{} --config-file {}\n".format(clientExe, configFile)) runScriptFile.write("ERR2=$?\n\n") runScriptFile.write(""" @@ -309,7 +306,7 @@ def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cC runScriptFile.write("%s -d 0 --setfan 50\n" % globalParameters["ROCmSMIPath"]) else: for configFile in configPaths: - runScriptFile.write("{} --config-file {} {} --best-solution 1\n".format(ClientExecutable.getClientExecutable(cxxCompiler, cCompiler, buildDir), configFile, globalParameters["ClientArgs"])) + runScriptFile.write("{} --config-file {} --best-solution 1\n".format(ClientExecutable.getClientExecutable(cxxCompiler, cCompiler, buildDir), configFile)) if os.name != "nt": runScriptFile.write("exit $ERR\n") runScriptFile.close() diff --git a/tensilelite/Tensile/Common/Constants.py b/tensilelite/Tensile/Common/Constants.py index 3df7457b0a..eb54ff4252 100644 --- a/tensilelite/Tensile/Common/Constants.py +++ b/tensilelite/Tensile/Common/Constants.py @@ -1,3 +1,13 @@ +# Subdirectories for Tensile build artifacts +CLIENT_BUILD_DIR: str = "0_Build" +BENCHMARK_PROBLEMS_DIR: str = "1_BenchmarkProblems" +BENCHMARK_DATA_DIR: str = "2_BenchmarkData" +LIBRARY_LOGIC_DIR: str = "3_LibraryLogic" +LIBRARY_CLIENT_DIR: str = "4_LibraryClient" + +# If a file name is longer than this value, and `--short-filenames` is set, shorten it with a hash. +MAX_FILENAME_LENGTH: int = 64 + # Characters to use for GEMM naming, e.g., C[ij]=Sum[k] A[ik]*B[jk] INDEX_CHARS: str = "IJKLMNOPQRSTUVWXYZ" diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 881932e292..eda9f6ea44 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -60,7 +60,6 @@ globalParameters["CodeFromFiles"] = ( True # if False byte arrays will be generated during Benchmarking phase as before ) -globalParameters["SortProblems"] = False # sort problems by size; else use order in YAML file globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't globalParameters["HardwareMonitor"] = ( True # False: disable benchmarking client monitoring clocks using rocm-smi. @@ -87,10 +86,6 @@ # Medium size: 0.75 # Large size : 0.9 -# cProfile -globalParameters["Profiler"] = ( - 0 # Enable profiler. 0=off, 1=cProfile. This will set CpuThreads to 1. -) # validation globalParameters["NumElementsToValidate"] = ( 128 # number of elements to validate, 128 will be evenly spaced out (with prime number stride) across C tensor @@ -123,13 +118,7 @@ globalParameters["SolutionSelectionAlg"] = ( 1 # algorithm to determine which solutions to keep. 0=removeLeastImportantSolutions, 1=keepWinnerSolutions (faster) ) -globalParameters["ExpandRanges"] = ( - True # expand ranges into exact configs before writing logic file. False ignores ranges. -) globalParameters["GenerateSourcesAndExit"] = False # Exit after kernel source generation. -globalParameters["WavefrontWidth"] = ( - 64 # if False and library client already built, then building library client will be skipped when tensile is re-run -) globalParameters["ExitOnFails"] = ( 1 # 1: Exit after benchmark run if failures detected. 2: Exit during benchmark run. ) @@ -271,9 +260,6 @@ False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids ) -globalParameters["MaxFileName"] = ( - 64 # If a file name would be longer than this, shorten it with a hash. -) globalParameters["SupportedISA"] = [ (8, 0, 3), (9, 0, 0), @@ -295,19 +281,6 @@ ] # assembly kernels writer supports these architectures globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. -globalParameters["ClientBuildPath"] = "0_Build" # subdirectory for host code build directory -globalParameters["BenchmarkProblemsPath"] = ( - "1_BenchmarkProblems" # subdirectory for benchmarking phases -) -globalParameters["BenchmarkDataPath"] = ( - "2_BenchmarkData" # subdirectory for storing final benchmarking data -) -globalParameters["LibraryLogicPath"] = ( - "3_LibraryLogic" # subdirectory for library logic produced by analysis -) -globalParameters["LibraryClientPath"] = ( - "4_LibraryClient" # subdirectory for building example library client -) globalParameters["ClientExecutionLockPath"] = ( None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. ) @@ -334,10 +307,6 @@ globalParameters["CodeObjectVersion"] = "4" globalParameters["Architecture"] = "all" -# might be deprecated -globalParameters["EnableHalf"] = False -globalParameters["ClientArgs"] = "" - # perf model globalParameters["PerfModelL2ReadHits"] = 0.0 globalParameters["PerfModelL2WriteHits"] = 0.15 @@ -374,7 +343,6 @@ # Mode 0 requires memcpy everytime when the problem changes to reset the data, but mode 1 doesn't. globalParameters["BuildIdKind"] = "sha1" -globalParameters["ValidateLibrary"] = False globalParameters["AsmDebug"] = ( False # Set to True to keep debug information for compiled code objects ) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 0491073a3d..a544e68aa6 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,7 +36,8 @@ from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule -from .Common import globalParameters, printWarning, roundUp, print2, printExit, INDEX_CHARS, DataDirection, SemanticVersion +from .Common import globalParameters, printWarning, roundUp, print2, printExit, DataDirection, SemanticVersion, \ + INDEX_CHARS, MAX_FILENAME_LENGTH import abc import os @@ -4947,13 +4948,13 @@ def _syncThreads(self, kernel, comment="", skipForceWaitcnt0=False): def _shortenFileBase(self, kernel): base = self.getKernelName(kernel) - if len(base) <= globalParameters["MaxFileName"]: + if len(base) <= MAX_FILENAME_LENGTH: return base import hashlib import base64 - pivot = globalParameters["MaxFileName"] * 3 // 4 + pivot = MAX_FILENAME_LENGTH * 3 // 4 firstPart = base[:pivot] secondPart = base[pivot:] diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 114501cfe4..c8de72ff2a 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -24,7 +24,8 @@ from pathlib import Path from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ - assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath + assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -317,30 +318,9 @@ def __init__(self, problemType, problemSizesList, solutionsList, \ #FIXME-problem self.rangeProblemSizes.update([tuple(problem.sizes) for problem in problemSizes.problems]) for rangeSize in problemSizes.ranges: - - if globalParameters["ExpandRanges"]: - # Treat ranges as pile of exacts: - for rsize in rangeSize.problemSizes: - self.exactProblemSizes.add(tuple(rsize)) - else: - # Create the ranges info in the logic file - #print "RangeSize", rangeSize - sizedIdx = 0 - mappedIdx = 0 - for i in range(0, self.numIndices): - if rangeSize.indexIsSized[i]: - index = rangeSize.indicesSized[sizedIdx] - sizedIdx += 1 - else: - index = rangeSize.indicesSized[ \ - rangeSize.indicesMapped[mappedIdx]] - mappedIdx += 1 - currentSize = index[0] - currentStride = index[1] - while currentSize <= index[3]: - unifiedProblemSizes[i].add(currentSize) - currentSize += currentStride - currentStride += index[2] + # Treat ranges as pile of exacts: + for rsize in rangeSize.problemSizes: + self.exactProblemSizes.add(tuple(rsize)) for i in range(0, len(unifiedProblemSizes)): unifiedProblemSizes[i] = sorted(list(unifiedProblemSizes[i])) print2("UnifiedProblemSizes: %s" % unifiedProblemSizes) @@ -1567,6 +1547,6 @@ def read_max_freq(): ################################################################################ ################################################################################ def main(config, cxxCompiler: str, outputPath: Path): - benchmarkDataPath = outputPath / globalParameters["BenchmarkDataPath"] - libraryLogicPath = outputPath / globalParameters["LibraryLogicPath"] + benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR + libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 52b0c23ba1..c374c13932 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -889,10 +889,7 @@ def __init__(self, problemType, config): self.problems.update({Problem(rangeSize) : 1}) for e in self.exacts: self.problems.update({e : 1}) - if globalParameters["SortProblems"]: - self.problems = sorted(list( self.problems.keys()), key=operator.attrgetter("sizes")) - else: - self.problems = list(self.problems.keys()) + self.problems = list(self.problems.keys()) self.totalProblemSizes = len(self.problems) # max sizes diff --git a/tensilelite/Tensile/Source/TensileCreateLibrary.cmake b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake index 33c3df2565..4df1e5124c 100644 --- a/tensilelite/Tensile/Source/TensileCreateLibrary.cmake +++ b/tensilelite/Tensile/Source/TensileCreateLibrary.cmake @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,6 @@ function(TensileCreateLibraryCmake Tensile_ARCHITECTURE Tensile_LIBRARY_FORMAT Tensile_SHORT_FILE_NAMES - Tensile_LIBRARY_PRINT_DEBUG Tensile_CPU_THREADS Tensile_SEPARATE_ARCHITECTURES Tensile_LAZY_LIBRARY_LOADING, diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 4c1d651641..81afc23b3b 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -31,10 +31,11 @@ import sys import argparse from .Common import globalParameters, print1, printExit, printWarning, ensurePath, \ - assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__ + assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain from .Toolchain.Validators import validateToolchain, ToolchainDefaults +from .Utilities.Decorators.Profile import profile from . import BenchmarkProblems from . import ClientWriter from . import LibraryIO @@ -52,6 +53,7 @@ # LibraryLogic.main() to analyse final benchmark data and produce logic/yaml # ClientWriter.main() to create client which calls library based on above yaml ################################################################################ +@profile def executeStepsInConfig( config: dict, outputPath: Path, @@ -88,7 +90,7 @@ def executeStepsInConfig( ############################################################################## # Library Logic ############################################################################## - libraryLogicDataPath = os.path.join(outputPath, globalParameters["LibraryLogicPath"]) + libraryLogicDataPath = os.path.join(outputPath, LIBRARY_LOGIC_DIR) if "LibraryLogic" in config: if os.path.exists(libraryLogicDataPath): libraryLogicFiles = os.listdir(libraryLogicDataPath) @@ -158,7 +160,6 @@ def splitExtraParameters(par): action="store", default="yaml", help="select which logic format to use") argParser.add_argument("--library-format", dest="LibraryFormat", choices=["yaml", "msgpack"], \ action="store", default="yaml", help="select which library format to use") - argParser.add_argument("--client-build-path", default=None) argParser.add_argument("--client-lock", default=None) argParser.add_argument("--prebuilt-client", default=None) @@ -192,8 +193,6 @@ def argUpdatedGlobalParameters(args): rv["CMakeBuildType"] = "Debug" if args.shortNames: rv["ShortNames"] = True - if args.client_build_path: - rv["ClientBuildPath"] = args.client_build_path if args.client_lock: rv["ClientExecutionLockPath"] = args.client_lock if args.prebuilt_client: @@ -423,24 +422,11 @@ def Tensile(userArgs): print("Overriding {0}={1}".format(key, value)) globalParameters[key] = value - # Enable profiler - profiler = None - if globalParameters["Profiler"] == 1: - printWarning("cProfiler is enabled. CpuThreads will be set to 1.") - globalParameters["CpuThreads"] = 1 - import cProfile - profiler = cProfile.Profile() - profiler.enable() + if "MaxFileName" in globalParameters or "MaxFileName" in config: + printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler) - if profiler: - profiler.disable() - filename = outputPath / "tensile.stats" - profiler.dump_stats(filename) - filename = outputPath / "tensile.prof" - profiler.dump_stats(filename) - def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/Tests/build_client.yaml b/tensilelite/Tensile/Tests/build_client.yaml index ce8f6956c1..cb38e7756d 100644 --- a/tensilelite/Tensile/Tests/build_client.yaml +++ b/tensilelite/Tensile/Tests/build_client.yaml @@ -8,7 +8,6 @@ GlobalParameters: CMakeBuildType: Release EnqueuesPerSync: 1 SyncsPerBenchmark: 1 - LibraryPrintDebug: False NumElementsToValidate: 0 ValidationMaxToPrint: 4 ValidationPrintValids: False diff --git a/tensilelite/Tensile/Tests/conftest.py b/tensilelite/Tensile/Tests/conftest.py index 0bcf40e049..894510feef 100644 --- a/tensilelite/Tensile/Tests/conftest.py +++ b/tensilelite/Tensile/Tests/conftest.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -105,7 +105,6 @@ def tensile_args(pytestconfig, builddir, worker_lock_path): if pytestconfig.getoption("--global-parameters"): rv += ["--global-parameters", pytestconfig.getoption("--global-parameters")] if not pytestconfig.getoption("--no-common-build"): - rv += ["--client-build-path", builddir] if pytestconfig.getoption("--prebuilt-client"): rv += ["--prebuilt-client", pytestconfig.getoption("--prebuilt-client")] From dc7cf35bc8f493054c485268b7970534448ce7f4 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 10 Feb 2025 17:01:01 -0700 Subject: [PATCH 034/105] Update preferred search paths (#4) --- tensilelite/Tensile/Toolchain/Validators.py | 143 +++++++++++++++----- 1 file changed, 107 insertions(+), 36 deletions(-) diff --git a/tensilelite/Tensile/Toolchain/Validators.py b/tensilelite/Tensile/Toolchain/Validators.py index a67c9bb7e1..5c2381a652 100644 --- a/tensilelite/Tensile/Toolchain/Validators.py +++ b/tensilelite/Tensile/Toolchain/Validators.py @@ -28,33 +28,80 @@ from typing import List, NamedTuple, Union from subprocess import run, PIPE -ROCM_BIN_PATH = Path("/opt/rocm/bin") -ROCM_LLVM_BIN_PATH = Path("/opt/rocm/lib/llvm/bin") +DEFAULT_ROCM_BIN_PATH_POSIX = Path("/opt/rocm/bin") +DEFAULT_ROCM_LLVM_BIN_PATH_POSIX = Path("/opt/rocm/lib/llvm/bin") +DEFAULT_ROCM_BIN_PATH_WINDOWS = Path("C:/Program Files/AMD/ROCm") -if os.name == "nt": - def _windowsLatestRocmBin(path: Union[Path, str]) -> Path: - """Get the path to the latest ROCm bin directory, on Windows. - This function assumes that ROCm versions are differentiated with the form ``X.Y``. +osSelect = lambda linux, windows: linux if os.name != "nt" else windows - Args: - path: The path to the ROCm root directory, typically ``C:/Program Files/AMD/ROCm``. - Returns: - The path to the ROCm bin directory for the latest ROCm version. - Typically of the form ``C:/Program Files/AMD/ROCm/X.Y/bin``. - """ - path = Path(path) - pattern = re.compile(r"^\d+\.\d+$") - versions = filter(lambda d: d.is_dir() and pattern.match(d.name), path.iterdir()) - latest = max(versions, key=lambda d: tuple(map(int, d.name.split(".")))) - return latest / "bin" - # LLVM binaries are in the same directory as ROCm binaries on Windows - ROCM_BIN_PATH = _windowsLatestRocmBin("C:/Program Files/AMD/ROCm") - ROCM_LLVM_BIN_PATH = _windowsLatestRocmBin("C:/Program Files/AMD/ROCm") +def _windowsLatestRocmBin(path: Union[Path, str]) -> Path: + """Get the path to the latest ROCm bin directory, on Windows. + This function assumes that ROCm versions are differentiated with the form ``X.Y``. -osSelect = lambda linux, windows: linux if os.name != "nt" else windows + Args: + path: The path to the ROCm root directory, typically ``C:/Program Files/AMD/ROCm``. + + Returns: + The path to the ROCm bin directory for the latest ROCm version. + Typically of the form ``C:/Program Files/AMD/ROCm/X.Y/bin``. + """ + path = Path(path) + pattern = re.compile(r"^\d+\.\d+$") + versions = filter(lambda d: d.is_dir() and pattern.match(d.name), path.iterdir()) + latest = max(versions, key=lambda d: tuple(map(int, d.name.split(".")))) + return latest / "bin" + + +def _windowsSearchPaths() -> List[Path]: + defaultPath = DEFAULT_ROCM_BIN_PATH_WINDOWS + searchPaths = [] + + if os.environ.get("HIP_PATH"): + hipPaths = [Path(p) / "bin" for p in os.environ["HIP_PATH"].split(os.pathsep)] + searchPaths.extend(hipPaths) + + if Path(defaultPath).exists(): + searchPaths.append(_windowsLatestRocmBin(defaultPath)) + + if os.environ.get("PATH"): + envPath = [Path(p) for p in os.environ["PATH"].split(os.pathsep)] + searchPaths.extend(envPath) + + return searchPaths + + +def _windowsWithExtensions(exe: str) -> List[str]: + if not os.name == "nt": + raise ValueError("These extensions should not be added on anything but Windows") + files = [exe] + files.extend([exe + ext.lower() for ext in os.environ["PATHEXT"].split(";")]) + return files + + +def _posixSearchPaths() -> List[Path]: + + searchPaths = [] + + if os.environ.get("ROCM_PATH"): + for p in os.environ["ROCM_PATH"].split(os.pathsep): + searchPaths.append(Path(p) / "bin") + searchPaths.append(Path(p) / "lib" / "llvm" / "bin") + + searchPaths.extend( + [ + DEFAULT_ROCM_BIN_PATH_POSIX, + DEFAULT_ROCM_LLVM_BIN_PATH_POSIX, + ] + ) + + if os.environ.get("PATH"): + envPath = [Path(p) for p in os.environ["PATH"].split(os.pathsep)] + searchPaths.extend(envPath) + + return searchPaths class ToolchainDefaults(NamedTuple): @@ -66,6 +113,8 @@ class ToolchainDefaults(NamedTuple): def _supportedComponent(component: str, targets: List[str]) -> bool: + if os.name == "nt": + targets = [tExt for t in targets for tExt in _windowsWithExtensions(t)] isSupported = any([component == t for t in targets]) or any([Path(component).name == t for t in targets]) return isSupported @@ -79,7 +128,7 @@ def supportedCCompiler(compiler: str) -> bool: Return: If supported True; otherwise, False. """ - return _supportedComponent(compiler, [ToolchainDefaults.C_COMPILER]) + return _supportedComponent(compiler, ["amdclang", "clang"]) def supportedCxxCompiler(compiler: str) -> bool: @@ -91,7 +140,7 @@ def supportedCxxCompiler(compiler: str) -> bool: Return: If supported True; otherwise, False. """ - return _supportedComponent(compiler, [ToolchainDefaults.CXX_COMPILER]) + return _supportedComponent(compiler, ["amdclang++", "clang++"]) def supportedOffloadBundler(bundler: str) -> bool: @@ -103,19 +152,33 @@ def supportedOffloadBundler(bundler: str) -> bool: Return: If supported True; otherwise, False. """ - return _supportedComponent(bundler, [ToolchainDefaults.OFFLOAD_BUNDLER]) + return _supportedComponent(bundler, ["clang-offload-bundler"]) -def supportedHip(smi: str) -> bool: - """Determine if an offload bundler is supported by Tensile. +def supportedHip(hip: str) -> bool: + """Determine if a hip callable binary is supported by Tensile. Args: - bundler: The name of an offload bundler to test for support. + hip: The name of an offload bundler to test for support. + + Return: + If supported True; otherwise, False. + """ + return _supportedComponent(hip, ["hipcc", "hipconfig"]) + + +def supportedDeviceEnumerator(enumerator: str) -> bool: + """Determine if a device enumerator is supported by Tensile. + + Args: + enumerator: The name of a device enumerator to test for support. Return: If supported True; otherwise, False. """ - return _supportedComponent(smi, [ToolchainDefaults.HIP_CONFIG]) + if os.name == "nt": + return _supportedComponent(enumerator, ["hipinfo", "hipInfo"]) + return _supportedComponent(enumerator, ["rocm_agent_enumerator", "amdgpu-arch"]) def _exeExists(file: Path) -> bool: @@ -141,14 +204,25 @@ def _validateExecutable(file: str, searchPaths: List[Path]) -> str: The validated executable with an absolute path. """ if not any(( - supportedCxxCompiler(file), supportedCCompiler(file), supportedOffloadBundler(file), supportedHip(file) + supportedCxxCompiler(file), + supportedCCompiler(file), + supportedOffloadBundler(file), + supportedHip(file), + supportedDeviceEnumerator(file) )): raise ValueError(f"{file} is not a supported toolchain component for OS: {os.name}") - if _exeExists(Path(file)): return file + # Check if the file is an absolute path and executable + if _exeExists(Path(file)): + return file + + # Then check the search paths + files = _windowsWithExtensions(file) if os.name == "nt" else [file] for path in searchPaths: - path /= file - if _exeExists(path): return str(path) + for f in files: + p = path / f + if _exeExists(p): + return str(p) raise FileNotFoundError(f"`{file}` either not found or not executable in any search path: {':'.join(map(str, searchPaths))}") @@ -168,10 +242,7 @@ def validateToolchain(*args: str): if not args: raise ValueError("No toolchain components to validate, at least one argument is required") - searchPaths = [ - ROCM_BIN_PATH, - ROCM_LLVM_BIN_PATH, - ] + [Path(p) for p in os.environ["PATH"].split(os.pathsep)] + searchPaths = _windowsSearchPaths() if os.name == "nt" else _posixSearchPaths() out = (_validateExecutable(x, searchPaths) for x in args) return next(out) if len(args) == 1 else tuple(out) From cf7b74ec44ebae3a1f48edaa7ad300d2c3660c27 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 10 Feb 2025 18:28:10 -0600 Subject: [PATCH 035/105] fix: remove files updated by precommit in merge --- clients/include/TensorDataManipulation.hpp | 2 +- clients/include/hipblaslt_arguments.hpp | 2 +- clients/include/hipblaslt_common.yaml | 3 ++- .../sample_hipblaslt_gemm_bias_swizzle_a_ext.cpp | 4 ++-- .../26_gemm_swizzle_a/sample_hipblaslt_gemm_swizzle_a.cpp | 2 +- .../sample_hipblaslt_weight_swizzle_padding.cpp | 4 ++-- clients/samples/CMakeLists.txt | 6 +++--- docs/api-reference.rst | 6 +++--- library/include/hipblaslt.h | 2 +- library/include/hipblaslt_float8.h | 2 +- library/include/hipblaslt_float8_bc.h | 2 +- .../rocblaslt/src/include/rocblaslt_mat_utils.hpp | 2 +- .../src/amd_detail/rocblaslt/src/include/tensile_host.hpp | 2 +- .../src/amd_detail/rocblaslt/src/rocblaslt_auxiliary.cpp | 2 +- library/src/amd_detail/rocblaslt/src/rocblaslt_mat.cpp | 2 +- library/src/amd_detail/rocblaslt/src/tensile_host.cpp | 2 +- 16 files changed, 23 insertions(+), 22 deletions(-) diff --git a/clients/include/TensorDataManipulation.hpp b/clients/include/TensorDataManipulation.hpp index 50602fae8a..227a9d5146 100644 --- a/clients/include/TensorDataManipulation.hpp +++ b/clients/include/TensorDataManipulation.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2024-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/clients/include/hipblaslt_arguments.hpp b/clients/include/hipblaslt_arguments.hpp index 1a3287bdc7..186435d64e 100644 --- a/clients/include/hipblaslt_arguments.hpp +++ b/clients/include/hipblaslt_arguments.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/clients/include/hipblaslt_common.yaml b/clients/include/hipblaslt_common.yaml index c79891e9ab..6faf371d01 100644 --- a/clients/include/hipblaslt_common.yaml +++ b/clients/include/hipblaslt_common.yaml @@ -53,7 +53,7 @@ Datatypes: d: 3 - hipblaslt_scaling_format: bases: [ c_int ] - attr: + attr: none: 0 Scalar: 1 Vector: 2 @@ -496,3 +496,4 @@ Defaults: compute_input_typeA: hipblaslt_datatype_invalid compute_input_typeB: hipblaslt_datatype_invalid scale_type: hipblaslt_datatype_invalid + \ No newline at end of file diff --git a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_bias_swizzle_a_ext.cpp b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_bias_swizzle_a_ext.cpp index a3c390663c..c2b124de60 100644 --- a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_bias_swizzle_a_ext.cpp +++ b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_bias_swizzle_a_ext.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2024-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -239,4 +239,4 @@ void swizzleGemmEpilogueBiasVecExt(hipblasLtHandle_t handle, CHECK_HIPBLASLT_ERROR(hipblasLtMatmulDescDestroy(matmul)); CHECK_HIP_ERROR(hipFree(d_bias)); return; -} +} \ No newline at end of file diff --git a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_swizzle_a.cpp b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_swizzle_a.cpp index 1adf61113f..95a53b7a67 100644 --- a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_swizzle_a.cpp +++ b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_gemm_swizzle_a.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2024-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_weight_swizzle_padding.cpp b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_weight_swizzle_padding.cpp index 632016c0fc..d73d336434 100644 --- a/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_weight_swizzle_padding.cpp +++ b/clients/samples/26_gemm_swizzle_a/sample_hipblaslt_weight_swizzle_padding.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2024-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -61,4 +61,4 @@ int main(int argc, char **argv) std::cout << "Swizzle weight:\n"; Tensor::Manipulation::printTensorDataMultiDims(std::cout, permuted); return 0; -} +} \ No newline at end of file diff --git a/clients/samples/CMakeLists.txt b/clients/samples/CMakeLists.txt index 91c86afbf8..3277eea72b 100644 --- a/clients/samples/CMakeLists.txt +++ b/clients/samples/CMakeLists.txt @@ -1,5 +1,5 @@ # ######################################################################## -# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. +# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -151,7 +151,7 @@ endforeach( ) foreach( exe ${samples} ) install( TARGETS ${exe} - RUNTIME + RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/hipblaslt-samples COMPONENT samples ) @@ -163,7 +163,7 @@ install( ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION ${CMAKE_INSTALL_DATADIR}/hipblaslt/samples COMPONENT samples - FILES_MATCHING + FILES_MATCHING PATTERN "*.cpp" PATTERN "*.h" PATTERN "CMakeLists.txt" EXCLUDE diff --git a/docs/api-reference.rst b/docs/api-reference.rst index 00d87e637f..1e2f770613 100644 --- a/docs/api-reference.rst +++ b/docs/api-reference.rst @@ -92,13 +92,13 @@ HIPBLAS_COMPUTE_32F HIP_R_32F HIP_R_16BF HIP_R_16BF For FP8 type Matmul, hipBLASLt supports the type combinations shown in the following table: -* This table uses simpler brieviations: +* This table uses simpler brieviations: + **FP16** means **HIP_R_16F** + **BF16** means **HIP_R_16BF** + **FP32** means **HIP_R_32F** - + **FP8** means **HIP_R_8F_E4M3_FNUZ** and - + **BF8** means **HIP_R_8F_E5M2_FNUZ** + + **FP8** means **HIP_R_8F_E4M3_FNUZ** and + + **BF8** means **HIP_R_8F_E5M2_FNUZ** * This table applies to all tranpose types (NN/NT/TT/TN) * **Default Bias Type** means the type when users don't explicitly specify the bias type diff --git a/library/include/hipblaslt.h b/library/include/hipblaslt.h index 21ad27b89d..09f586c804 100644 --- a/library/include/hipblaslt.h +++ b/library/include/hipblaslt.h @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/include/hipblaslt_float8.h b/library/include/hipblaslt_float8.h index 32d1fa2384..11f1e0dbca 100644 --- a/library/include/hipblaslt_float8.h +++ b/library/include/hipblaslt_float8.h @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2019-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/include/hipblaslt_float8_bc.h b/library/include/hipblaslt_float8_bc.h index 3a8853deb3..ea5e21da81 100644 --- a/library/include/hipblaslt_float8_bc.h +++ b/library/include/hipblaslt_float8_bc.h @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2019-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2019-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/include/rocblaslt_mat_utils.hpp b/library/src/amd_detail/rocblaslt/src/include/rocblaslt_mat_utils.hpp index 4495b84172..a3a22a1921 100644 --- a/library/src/amd_detail/rocblaslt/src/include/rocblaslt_mat_utils.hpp +++ b/library/src/amd_detail/rocblaslt/src/include/rocblaslt_mat_utils.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/include/tensile_host.hpp b/library/src/amd_detail/rocblaslt/src/include/tensile_host.hpp index 8f9acbf87e..6dee1f28f9 100644 --- a/library/src/amd_detail/rocblaslt/src/include/tensile_host.hpp +++ b/library/src/amd_detail/rocblaslt/src/include/tensile_host.hpp @@ -3,7 +3,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/rocblaslt_auxiliary.cpp b/library/src/amd_detail/rocblaslt/src/rocblaslt_auxiliary.cpp index 114d387bcd..f54455aa48 100644 --- a/library/src/amd_detail/rocblaslt/src/rocblaslt_auxiliary.cpp +++ b/library/src/amd_detail/rocblaslt/src/rocblaslt_auxiliary.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/rocblaslt_mat.cpp b/library/src/amd_detail/rocblaslt/src/rocblaslt_mat.cpp index cd9fb03d81..4e9f60a328 100644 --- a/library/src/amd_detail/rocblaslt/src/rocblaslt_mat.cpp +++ b/library/src/amd_detail/rocblaslt/src/rocblaslt_mat.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp index f25dc758a3..bee0ee9fa6 100644 --- a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp +++ b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal From 9de3e29c059463d76cdd5798aabc810f653b3270 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 10 Feb 2025 18:30:05 -0600 Subject: [PATCH 036/105] fix: remove log file --- tensilelite/Tensile/tcl.log | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tensilelite/Tensile/tcl.log diff --git a/tensilelite/Tensile/tcl.log b/tensilelite/Tensile/tcl.log deleted file mode 100644 index 1141235085..0000000000 --- a/tensilelite/Tensile/tcl.log +++ /dev/null @@ -1 +0,0 @@ -zsh: no such file or directory: Tensile/bin/TensileCreateLibrary From 8f483438dcb27b507d5a338a30e9e38f425c39c5 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 15:12:10 +0000 Subject: [PATCH 037/105] Copyright update --- tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp | 2 +- tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp | 2 +- tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp | 2 +- .../Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp | 2 +- tensilelite/Tensile/cmake/TensileConfigVersion.cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp b/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp index 45e37fc83c..6ebbb749bd 100644 --- a/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp +++ b/tensilelite/Tensile/Source/client/source/HardwareMonitor.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp b/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp index f17318933f..00f0cdf3b2 100644 --- a/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp +++ b/tensilelite/Tensile/Source/client/source/ResultFileReporter.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp b/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp index 39d6be20ea..3a766b2786 100644 --- a/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp +++ b/tensilelite/Tensile/Source/lib/source/TensorDescriptor.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp b/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp index b899654668..17e501b921 100644 --- a/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp +++ b/tensilelite/Tensile/Source/lib/source/hip/HipSolutionAdapter.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake index 13e3b920e4..2a24d18c48 100644 --- a/tensilelite/Tensile/cmake/TensileConfigVersion.cmake +++ b/tensilelite/Tensile/cmake/TensileConfigVersion.cmake @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 2ebb6ea6a4f7ad6844f15130f8e4e4863cbc0b1b Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 16:05:26 +0000 Subject: [PATCH 038/105] Remove short names from globalParameters --- tensilelite/Tensile/BenchmarkProblems.py | 14 +++++++------ tensilelite/Tensile/ClientWriter.py | 6 +++--- .../Tensile/Common/GlobalParameters.py | 3 --- tensilelite/Tensile/KernelWriter.py | 9 +++++---- tensilelite/Tensile/KernelWriterAssembly.py | 4 ++-- tensilelite/Tensile/Tensile.py | 7 ++++--- .../Tensile/TensileCreateLibrary/Run.py | 20 ++++++++++--------- tensilelite/Tensile/Toolchain/Assembly.py | 7 ++++--- 8 files changed, 37 insertions(+), 33 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 350ab1fa56..1241a61346 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -113,7 +113,8 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path): + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, + useShortNames: bool): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -150,7 +151,8 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ sourcePath, asmToolchain, srcToolchain, \ solutions, kernels, kernelHelperObjs, \ kernelWriterAssembly, errorTolerant=True, fromTensile=True, \ - generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"]) + generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], \ + useShortNames=useShortNames) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') @@ -200,7 +202,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -314,7 +316,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath) + sourcePath, useShortNames) # ^ this mutates solutions # write cache data @@ -379,7 +381,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG return (resultsFileBaseFinal, benchmarkTestFails) -def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path): +def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) @@ -418,7 +420,7 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # benchmark problem size group benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath) + benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath, useShortNames) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 5d3e31367d..aa5aac2f1b 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -97,7 +97,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): functions = [] functionNames = [] - createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler) + createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler, config["ShortNames"]) subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) coList = glob(os.path.join(clientLibraryPath, "library/*.co")) yamlList = glob(os.path.join(clientLibraryPath, "library/*.yaml")) @@ -200,7 +200,7 @@ def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: return process.returncode -def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): +def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, useShortNames: bool=False): import io runScriptFile = io.StringIO() @@ -209,7 +209,7 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): if not globalParameters["LazyLibraryLoading"]: callCreateLibraryCmd += " --no-lazy-library-loading" - if globalParameters["ShortNames"]: + if useShortNames: callCreateLibraryCmd += " --short-file-names" if globalParameters.get("AsmDebug", False): diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index eda9f6ea44..8d1570df1a 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -256,9 +256,6 @@ # shouldn't need to change globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use -globalParameters["ShortNames"] = ( - False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids -) globalParameters["SupportedISA"] = [ (8, 0, 3), diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index a544e68aa6..06711a4fc8 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -348,6 +348,7 @@ class ExternClasses: activation: ActivationModule = ActivationModule() biasSumUnroll: Optional[Component.SumUnroll] = None + ################################################################################ # Kernel Writer ################################################################################ @@ -4964,8 +4965,8 @@ def _shortenFileBase(self, kernel): return firstPart + secondPart - def _getCustomKernelSource(self, kernel, CustomKernelDirectory): - kernelName = self.getKernelFileBase(kernel) + def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): + kernelName = self.getKernelFileBase(useShortNames, kernel) with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: hipccver = globalParameters['HipClangVersion'].split(".") hipccMaj = int(hipccver[0]) @@ -5014,10 +5015,10 @@ def _getKernelSource(self, kernel: Solution): ############################################################################## # get kernel name ############################################################################## - def getKernelFileBase(self, kernel): + def getKernelFileBase(self, useShortNames: bool, kernel): if isCustomKernelConfig(kernel): fileBase = kernel["CustomKernelName"] - elif globalParameters["ShortNames"]: + elif useShortNames: fileBase = Solution.getNameSerial(kernel, self.kernelSerialNaming) else: fileBase = self._shortenFileBase(kernel) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 6ecfb82278..38b04a08c5 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -70,7 +70,7 @@ class KernelWriterAssembly(KernelWriter): def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion) - def getSourceFileString(self, kernel) -> Tuple[int, str]: + def getSourceFileString(self, kernel, useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" # Skip if .o files will have already been built for this file if kernel.duplicate: @@ -78,7 +78,7 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: return (0, "") # should this be an non zero number try: - code = self._getCustomKernelSource(kernel, CUSTOM_KERNEL_PATH) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) + code = self._getCustomKernelSource(useShortNames, kernel, CUSTOM_KERNEL_PATH) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) errcode = 0 except RuntimeError as e: printWarning(f"Failed to generate assembly source code for {kernel}: {e}") diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 81afc23b3b..3cc1ff2c18 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -84,7 +84,7 @@ def executeStepsInConfig( # Benchmark Problems ############################################################################## if "BenchmarkProblems" in config: - BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath) + BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath, config["ShortNames"]) print1("") ############################################################################## @@ -115,7 +115,7 @@ def executeStepsInConfig( libraryClientConfig = config["LibraryClient"] else: libraryClientConfig = {} - ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath) + ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath, config["ShortNames"]) print1("") @@ -412,12 +412,13 @@ def Tensile(userArgs): cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) - asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) overrideParameters = argUpdatedGlobalParameters(args) + config["ShortNames"] = args["ShortNames"] + for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) globalParameters[key] = value diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 445037c653..6687b739d1 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -81,15 +81,15 @@ class KernelCodeGenResult(NamedTuple): wavefrontSize: int -def processKernelSource(kernelWriterAssembly, ti, kernel) -> KernelCodeGenResult: +def processKernelSource(kernelWriterAssembly, ti, useShortNames, kernel) -> KernelCodeGenResult: """ Generate source for a single kernel. Returns (error, source, header, kernelName). """ kernelWriter = kernelWriterAssembly kernelWriter.setTensileInstructions(ti) - asmFilename = kernelWriter.getKernelFileBase(kernel) - err, src = kernelWriter.getSourceFileString(kernel) + asmFilename = kernelWriter.getKernelFileBase(useShortNames, kernel) + err, src = kernelWriter.getSourceFileString(kernel, useShortNames) header = kernelWriter.getHeaderFileString(kernel) objFilename = kernel._state.get("codeObjectFile", None) @@ -201,6 +201,7 @@ def writeSolutionsAndKernels( generateSourcesAndExit=False, compress=True, fromTensile=False, + useShortNames=False, ): codeObjectFiles = [] @@ -221,7 +222,7 @@ def writeSolutionsAndKernels( visited = set() duplicates = 0 for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) + base = kernelWriterAssembly.getKernelFileBase(useShortNames, k) k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -232,7 +233,7 @@ def writeSolutionsAndKernels( numKernels = len(asmKernels) assert numKernels == numAsmKernels, "Only assembly kernels are supported in TensileLite" asmIter = zip( - itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), asmKernels + itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), itertools.repeat(useShortNames), asmKernels ) asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels") removeInvalidSolutionsAndKernels( @@ -258,7 +259,7 @@ def assemble(ret): if not generateSourcesAndExit: codeObjectFiles += buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames ) buildSourceCodeObjectFiles( srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile @@ -276,6 +277,7 @@ def writeSolutionsAndKernelsTCL( kernelWriterAssembly, compress=True, fromTensile=False, + useShortNames=False, ): outputPath = Path(outputPath) @@ -295,7 +297,7 @@ def writeSolutionsAndKernelsTCL( visited = set() duplicates = 0 for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) + base = kernelWriterAssembly.getKernelFileBase(useShortNames, k) k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -309,7 +311,7 @@ def assemble(ret): asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) unaryProcessKernelSource = functools.partial( - processKernelSource, kernelWriterAssembly, TensileInstructions() + processKernelSource, kernelWriterAssembly, TensileInstructions(), useShortNames ) unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) @@ -320,7 +322,7 @@ def assemble(ret): multiArg=False, ) buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames ) writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index f009c63650..b986aee81c 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -175,7 +175,8 @@ def buildAssemblyCodeObjectFiles( writer: KernelWriterAssembly, destDir: Union[Path, str], asmDir: Union[Path, str], - compress: bool=True + compress: bool=True, + useShortNames: bool=False, ): """Builds code object files from assembly files @@ -208,14 +209,14 @@ def buildAssemblyCodeObjectFiles( gfx = isaToGfx(arch) - objectFiles = [str(asmDir / (writer.getKernelFileBase(k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] + objectFiles = [str(asmDir / (writer.getKernelFileBase(useShortNames, k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] coFileMap = collections.defaultdict(list) if len(objectFiles): coFileMap[asmDir / ("TensileLibrary_"+ gfx + extCoRaw)] = objectFiles for kernel in archKernels: coName = kernel.get("codeObjectFile", None) if coName: - coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(kernel) + extObj))) + coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(useShortNames, kernel) + extObj))) for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(objFiles, coFileRaw) toolchain.link(objFiles, str(coFileRaw)) From 66c734b636dfe5caa2f54c95d033aaf01c1123c1 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 20:45:46 +0000 Subject: [PATCH 039/105] Remove global params for kernel debugging --- tensilelite/Tensile/BenchmarkProblems.py | 39 ++++++----- .../Tensile/Common/GlobalParameters.py | 19 ------ tensilelite/Tensile/KernelWriter.py | 57 +++++++++-------- tensilelite/Tensile/SolutionStructs.py | 15 ----- tensilelite/Tensile/Tensile.py | 64 +++++++++++++------ .../Tensile/TensileCreateLibrary/Run.py | 1 + 6 files changed, 98 insertions(+), 97 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 1241a61346..584b243516 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -31,17 +31,18 @@ from copy import deepcopy from pathlib import Path -from . import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO -from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations -from .Contractions import ProblemType as ContractionsProblemType -from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni -from .KernelWriterAssembly import KernelWriterAssembly -from .SolutionStructs import Solution, ProblemType, ProblemSizes -from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels -from .CustomKernels import getCustomKernelConfig -from .Toolchain.Assembly import AssemblyToolchain -from .Toolchain.Source import SourceToolchain -from .Common import globalParameters, HR, print1, print2, \ +from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO +from Tensile.BenchmarkStructs import BenchmarkProcess, constructForkPermutations +from Tensile.Contractions import ProblemType as ContractionsProblemType +from Tensile.ClientWriter import runClient, writeClientConfig, writeClientConfigIni +from Tensile.KernelWriterAssembly import KernelWriterAssembly +from Tensile.KernelWriter import DebugConfig +from Tensile.SolutionStructs import Solution, ProblemType, ProblemSizes +from Tensile.TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels +from Tensile.CustomKernels import getCustomKernelConfig +from Tensile.Toolchain.Assembly import AssemblyToolchain +from Tensile.Toolchain.Source import SourceToolchain +from Tensile.Common import globalParameters, HR, print1, print2, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR @@ -114,7 +115,8 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, - useShortNames: bool): + useShortNames: bool, + dbConfig: DebugConfig): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -143,7 +145,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, dbConfig) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] @@ -202,7 +204,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, dbConfig: DebugConfig ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -316,7 +318,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath, useShortNames) + sourcePath, useShortNames, dbConfig) # ^ this mutates solutions # write cache data @@ -381,7 +383,8 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG return (resultsFileBaseFinal, benchmarkTestFails) -def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool): +def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, \ + cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool, dbConfig: DebugConfig): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) @@ -420,7 +423,9 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # benchmark problem size group benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath, useShortNames) + benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, \ + asmToolchain, srcToolchain, cCompiler, buildTmpPath, \ + benchmarkProblemsPath, useShortNames, dbConfig) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 8d1570df1a..0075ac2ec8 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -132,14 +132,6 @@ # the kernel. Useful to examine and debug overflow errors. globalParameters["ForceGenerateKernel"] = 0 -######################################## -# optimization knob controls -######################################## - -globalParameters["UnrollLoopEfficiencyEnable"] = ( - False # if True split(S) MAC&LDS in each unroll iteration into n smaller groups.. -) - ######################################## # less common ######################################## @@ -205,22 +197,11 @@ # build parameters globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake globalParameters["CMakeCFlags"] = "" # pass flags to cmake -globalParameters["DebugKernel"] = ( - False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed -) globalParameters["AsanBuild"] = False # build with asan globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp # debug for assembly -globalParameters["EnableAsserts"] = False # Enable assembly debug assert -globalParameters["EnableDebugA"] = False # Enable / Disable CheckValue1A -globalParameters["EnableDebugB"] = False # Enable / Disable CheckValue1B -globalParameters["EnableDebugC"] = False # Enable / Disable CheckValueC -globalParameters["ExpectedValueC"] = 16.0 # Expected C Value when CheckValueC, debug for Alpha*A*B -globalParameters["ForceCExpectedValue"] = ( - False # Force C to "DebugExpectedValueC", debug for global write -) globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM # Tensor printing controls: diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 06711a4fc8..4de70083c7 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -349,6 +349,15 @@ class ExternClasses: biasSumUnroll: Optional[Component.SumUnroll] = None +class DebugConfig(NamedTuple): + enableAsserts: bool=False + enableDebugA: bool=False + enableDebugB: bool=False + enableDebugC: bool=False + expectedValueC: float=16.0 + forceCExpectedValue: bool=False + debugKernel: bool=False + ################################################################################ # Kernel Writer ################################################################################ @@ -358,12 +367,13 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler self.amdClangVersion = amdClangVersion self.ti = None + self.debugConfig = debugConfig self.do = {} self.do["PreLoop"] = True @@ -393,7 +403,7 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # Various debug flags and modes self.db = {} - self.db["EnableAsserts"] = globalParameters["EnableAsserts"] # Enable assertion codegen. Requires 2 SGPR. + self.db["EnableAsserts"] = self.debugConfig.enableAsserts # Enable assertion codegen. Requires 2 SGPR. self.db["DebugKernelMaxItems"] = 16 # Capture first N(=16) print values, ignore subsequent. If -1, debug writing is faster but writing more than 16 values is undefined. # Chicken bit to add conservative synchronization at strategic points: @@ -420,8 +430,8 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # Requires DataInitTypeAB=1. # Only works if the problem uses full tiles (no edges) # Mismatches will assert (generate GPUVM fault) - self.db["CheckValue1A"] = globalParameters["EnableDebugA"] - self.db["CheckValue1B"] = globalParameters["EnableDebugB"] + self.db["CheckValue1A"] = self.debugConfig.enableDebugA + self.db["CheckValue1B"] = self.debugConfig.enableDebugB self.db["CheckValue1Metadata"] = False # Check value in C matrix. # Caveats: @@ -430,15 +440,15 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # - Only works if matrix is integral multiple of macro-tile (no edges) - check is dumb so doesn't know # which work-items are outside the valid edge. # - Does not work in OptNoLoadLoop - self.db["CheckValueC"] = globalParameters["EnableDebugC"] + self.db["CheckValueC"] = self.debugConfig.enableDebugC # value expected if CheckValueC is set. Use '.' for FP. # For example could be 16.0 if U=8 and alpha=2 - self.db["ValueCExpectedValue"] = globalParameters["ExpectedValueC"] + self.db["ValueCExpectedValue"] = self.debugConfig.expectedValueC # Force an expected value for all C outputs. # May be useful for checking store path # See same caveats as CheckValueC - self.db["ForceExpectedValue"] = globalParameters["ForceCExpectedValue"] + self.db["ForceExpectedValue"] = self.debugConfig.forceCExpectedValue # Force VSerial value into the output, this will # not match reference but can be useful to see which work-items are @@ -468,7 +478,7 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang self.db["PrintStoreRegisterDb"] = False self.dumpData = Dump("DebugKernelItems", "AddressDbg", self.db["DebugKernelMaxItems"], \ - globalParameters["DebugKernel"]) + self.debugConfig.debugKernel) self.labels = LabelManager() # KernelWriter values @@ -560,7 +570,7 @@ def makeSchedule(self, kernel, tensorParametersA, tensorParametersB, localWriteE # returns: a Module with the combined, optimally scheduled # localReadCode + otherCode ############################################################################## - def makeSubIterSchedule(self, kernel, tPA, tPB, localReadCode, iteration, pointerLWCode, pointerLRCode, waitCode, macIterCode, \ + def _makeSubIterSchedule(self, kernel, tPA, tPB, localReadCode, iteration, pointerLWCode, pointerLRCode, waitCode, macIterCode, \ waitLWCode = Module(), syncCode = Module(), packCode = Module(), prevIterCode = Module(), NLLlast = False): iterCode = Module() @@ -1446,17 +1456,12 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): # (which is always just before the macs) lgkmcnt += localWrites else: - # if UnrollLoopEfficiencyEnable == True use waitCode passed lgkmCnt - # else: # we need to wait for all preceding reads before the macs # so only opportunity for optimization is if the writes are at the end - if globalParameters["UnrollLoopEfficiencyEnable"]: - lgkmcnt = waitCode.lgkmcnt + if localReads: + lgkmcnt = 0 # reset to wait for all reads else: - if localReads: - lgkmcnt = 0 # reset to wait for all reads - else: - lgkmcnt = localWrites # this only survives if writes are at the end + lgkmcnt = localWrites # this only survives if writes are at the end waitCode.comment += " old=%u, new=%u newLW=%u newLR=%u" % (waitCode.lgkmcnt, lgkmcnt,localWrites,localReads) if iteration == 0: @@ -1919,7 +1924,7 @@ def noLoadLoopBody( self, kernel, tensorParametersA, tensorParametersB, pack, is tP = tensorParametersA if kernel["ProblemType"]["BiasSrc"] == "A" else tensorParametersB macIterCode.add(self.exclasses.biasSumUnroll.loopSum(self, kernel, tP, u, kernel["InnerUnroll"])) - subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module, NLLlast) module.add(subIterCode) pack[luIdx] = Module() @@ -2002,7 +2007,7 @@ def noLoadLoop( self, kernel, tensorParametersA, tensorParametersB, isOptNLL, is # dsWriteBA is to do ds_write B first. # grBA is to do buffer_load B first. ############################################################################## - def loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, firstIter=False, dsWriteBA=False, grBA=False, isDTVGRSecondBuf=False, skipClose=False ): + def _loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, firstIter=False, dsWriteBA=False, grBA=False, isDTVGRSecondBuf=False, skipClose=False ): module = Module("loopBody") expand = kernel["ExpandPointerSwap"] @@ -2319,8 +2324,8 @@ def loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loop ## 8x8 -> split into group of 16 MAC(s) ## supports only PLR=0 ############################################################################### - if self.states.numItersPLR or (not globalParameters["UnrollLoopEfficiencyEnable"]): - subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + if self.states.numItersPLR: + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) module.add(subIterCode) # add scheduled "other", local reads, local writes pack[luIdx] = Module() @@ -2507,7 +2512,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): dsWriteBA = True if isULSGRO else False # second GR buffer check for DTV isDTVGRSecondBuf = True if isDTV else False - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, 0, loopCopies, False , dsWriteBA=dsWriteBA, isDTVGRSecondBuf=isDTVGRSecondBuf, skipClose=True)) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, 0, loopCopies, False , dsWriteBA=dsWriteBA, isDTVGRSecondBuf=isDTVGRSecondBuf, skipClose=True)) loopLabelToNoGRloopAfterABLoop = Label("NoGRloopAfterABLoop", "" ) loopCounter = self.loopCounter(kernel, self.states.unrollIdx) module.add(SSubU32(dst=loopCounter, src0=loopCounter, \ @@ -2519,14 +2524,14 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.add(SCBranchSCC1(labelName=loopLabelToNoGRloopAfterABLoop.getLabelName(), comment="exit LoopL" )) # grBA check for UnrollLoopSwapGlobalReadOrder grBA = True if isULSGRO else False - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, 1, loopCopies, True , grBA=grBA)) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, 1, loopCopies, True , grBA=grBA)) else: for lc in range(0, loopCopies): # second GR buffer check for DTV isDTVGRSecondBuf = True if isDTV and lc == 0 else False # loop body code generation finalLoop = lc == loopCopies - 1 - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, isDTVGRSecondBuf=isDTVGRSecondBuf )) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, isDTVGRSecondBuf=isDTVGRSecondBuf )) module.addComment1("Before NLL: Check VGPR.checkin for INT8 LW") @@ -2882,7 +2887,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): ############################################################################## # Init Kernel ############################################################################## - def initKernel(self, kernel, tensorParametersA, tensorParametersB): + def _initKernel(self, kernel, tensorParametersA, tensorParametersB): assert kernel["KernelLanguage"] == "Assembly" self.language = "ASM" # ISA version, such as 803 @@ -4999,7 +5004,7 @@ def _getKernelSource(self, kernel: Solution): fileString = "" tensorParametersA = {} tensorParametersB = {} - self.initKernel(kernel, tensorParametersA, tensorParametersB) + self._initKernel(kernel, tensorParametersA, tensorParametersB) self.stringIdx = 0 (error, kb) = self.kernelBody(kernel, tensorParametersA, tensorParametersB) fileString += str(kb) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index c374c13932..8e9686cc71 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -4116,21 +4116,6 @@ def calcEpilogueTurns(factorDims: List) -> int: state["AssignedDerivedParameters"] = True - # UnrollLoopEfficiencyEnable does not work with f16/bf16/int8x4 - if globalParameters["UnrollLoopEfficiencyEnable"] and (state["ProblemType"]["DataType"].isHalf() or \ - state["ProblemType"]["DataType"].isBFloat16() or state["ProblemType"]["DataType"].isInt8x4()): - reject(state, "UnrollLoopEfficiencyEnable does not support f16/bf16/int8x4") - - # UnrollLoopEfficiencyEnable supports only ThreadTile0,1=[6,4] or [4,6] or [4,4] or [6.6] or [8,4] or [4,8] - if globalParameters["UnrollLoopEfficiencyEnable"] and \ - not ((state["ThreadTile0"] == 6 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 6) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 6 and state["ThreadTile1"] == 6) or \ - (state["ThreadTile0"] == 8 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 8)): - reject(state, "UnrollLoopEfficiencyEnable does not support ThreadTile0,1 = [%u,%u]"%(state["ThreadTile0"], state["ThreadTile1"])) - # Set E if state["ProblemType"]["UseE"]: if (state["_GlobalAccumulation"] == 'SingleBuffer') and state["GlobalSplitU"] > 1: diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 3cc1ff2c18..9082f9e26c 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -26,24 +26,25 @@ print("This file can no longer be run as a script. Run 'Tensile/bin/Tensile' instead.") exit(1) -import joblib -import os +import subprocess import sys import argparse -from .Common import globalParameters, print1, printExit, printWarning, ensurePath, \ - assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR -from .Toolchain.Assembly import AssemblyToolchain -from .Toolchain.Source import SourceToolchain -from .Toolchain.Validators import validateToolchain, ToolchainDefaults -from .Utilities.Decorators.Profile import profile -from . import BenchmarkProblems -from . import ClientWriter -from . import LibraryIO -from . import LibraryLogic + from datetime import datetime from pathlib import Path -import subprocess +from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ + assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR +from Tensile.Toolchain.Assembly import AssemblyToolchain +from Tensile.Toolchain.Source import SourceToolchain +from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults +from Tensile.Utilities.Decorators.Profile import profile +from Tensile import BenchmarkProblems +from Tensile import ClientWriter +from Tensile import LibraryIO +from Tensile import LibraryLogic + +from Tensile.KernelWriter import DebugConfig ############################################################################### # Execute Steps in Config @@ -59,7 +60,8 @@ def executeStepsInConfig( outputPath: Path, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, - cCompiler: str + cCompiler: str, + dbconfig: DebugConfig ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -84,7 +86,8 @@ def executeStepsInConfig( # Benchmark Problems ############################################################################## if "BenchmarkProblems" in config: - BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath, config["ShortNames"]) + BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, \ + cCompiler, outputPath, buildTmpPath, config["ShortNames"], dbconfig) print1("") ############################################################################## @@ -191,8 +194,6 @@ def argUpdatedGlobalParameters(args): print1("# Command-line override: Debug") rv["PrintLevel"] = 2 rv["CMakeBuildType"] = "Debug" - if args.shortNames: - rv["ShortNames"] = True if args.client_lock: rv["ClientExecutionLockPath"] = args.client_lock if args.prebuilt_client: @@ -305,6 +306,26 @@ def store_max_frequency(max_frequency): return False + def debugConfig(config: dict) -> DebugConfig: + dbConfig = DebugConfig() + + if "EnableAsserts" in config: + dbConfig.enableAsserts = config["EnableAsserts"] + if "EnableDebugA" in config: + dbConfig.enableDebugA = config["EnableDebugA"] + if "EnableDebugB" in config: + dbConfig.enableDebugB = config["EnableDebugB"] + if "EnableDebugC" in config: + dbConfig.enableDebugC = config["EnableDebugC"] + if "ExpectedValueC" in config: + dbConfig.expectedValueC = config["ExpectedValueC"] + if "ForceCExpectedValue" in config: + dbConfig.forceCExpectedValue = config["ForceCExpectedValue"] + if "DebugKernel" in config: + dbConfig.debugKernel = config["DebugKernel"] + return dbConfig + + ################################################################################ # Tensile # - below entry points call here @@ -416,8 +437,11 @@ def Tensile(userArgs): srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) overrideParameters = argUpdatedGlobalParameters(args) - - config["ShortNames"] = args["ShortNames"] + + if "ShortNames" not in config: + config["ShortNames"] = args["ShortNames"] + + dbConfig = debugConfig(config) for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) @@ -426,7 +450,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler) + executeStepsInConfig(config, dbConfig, outputPath, asmToolchain, srcToolchain, cCompiler) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 6687b739d1..29f47d4eef 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -591,6 +591,7 @@ def validLogicFile(p: Path): kernelHelperObjs, kernelWriterAssembly, compress=arguments["UseCompression"], + arguments["ShortNames"], ) archs = [ From f64db430fea3229f4e95cc1ea9e9ca5bec07e399 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 22:08:27 +0000 Subject: [PATCH 040/105] Resolve tox failures --- tensilelite/Tensile/BenchmarkProblems.py | 12 +++---- tensilelite/Tensile/Components/Signature.py | 6 ++-- tensilelite/Tensile/KernelWriter.py | 17 ++++----- tensilelite/Tensile/KernelWriterAssembly.py | 12 +++---- tensilelite/Tensile/Tensile.py | 35 ++++++++++--------- .../Tensile/TensileCreateLibrary/Run.py | 2 +- 6 files changed, 41 insertions(+), 43 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 584b243516..89f51a784d 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -116,7 +116,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, useShortNames: bool, - dbConfig: DebugConfig): + debugConfig: DebugConfig): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -145,7 +145,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, dbConfig) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, debugConfig) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] @@ -204,7 +204,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, dbConfig: DebugConfig + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, debugConfig: DebugConfig ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -318,7 +318,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath, useShortNames, dbConfig) + sourcePath, useShortNames, debugConfig) # ^ this mutates solutions # write cache data @@ -384,7 +384,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, \ - cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool, dbConfig: DebugConfig): + cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool, debugConfig: DebugConfig): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) @@ -425,7 +425,7 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source (resultsFileBaseFinal, benchmarkErrors) = \ benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, \ asmToolchain, srcToolchain, cCompiler, buildTmpPath, \ - benchmarkProblemsPath, useShortNames, dbConfig) + benchmarkProblemsPath, useShortNames, debugConfig) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/Components/Signature.py b/tensilelite/Tensile/Components/Signature.py index 778d0305f4..6964b35b75 100644 --- a/tensilelite/Tensile/Components/Signature.py +++ b/tensilelite/Tensile/Components/Signature.py @@ -23,7 +23,7 @@ ################################################################################ from ..Component import Signature -from ..Common import globalParameters, DataDirection +from ..Common import DataDirection from ..TensileInstructions import SignatureBase from ..TensileInstructions import SignatureValueKind as SVK from ..Activation import ActivationType @@ -118,7 +118,7 @@ def __call__(self, writer) -> SignatureBase: kernArgReg -= 2 # strides kernArgReg += kernel["ProblemType"]["NumIndicesSummation"] kernArgReg += kernel["ProblemType"]["NumIndicesC"] - if globalParameters["DebugKernel"]: + if writer.debugConfig.debugKernel: kernArgReg += writer.states.rpga # debug buffer # kernArgBytes = kernArgReg * 4 # bytes/reg @@ -159,7 +159,7 @@ def __call__(self, writer) -> SignatureBase: signature.addArg( "SizesSum%u"%i, SVK.SIG_VALUE, "u32") userArgumentsInfo.gemmArgumentSize += 4 - if globalParameters["DebugKernel"]: + if writer.debugConfig.debugKernel: signature.addArg("AddressDbg", SVK.SIG_GLOBALBUFFER, "struct", "generic") signature.addArg( "D", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") signature.addArg( "C", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 4de70083c7..39dcc1103e 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -2324,13 +2324,10 @@ def _loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loo ## 8x8 -> split into group of 16 MAC(s) ## supports only PLR=0 ############################################################################### - if self.states.numItersPLR: - subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ - u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) - module.add(subIterCode) # add scheduled "other", local reads, local writes - pack[luIdx] = Module() - else: - printExit("TensileLite does not support MAC instructions.") + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) + module.add(subIterCode) # add scheduled "other", local reads, local writes + pack[luIdx] = Module() # close unrolled loop endStr = "" @@ -3628,7 +3625,7 @@ def readWriteVectors(mat, vw, kernel): else: numVgprGlobalReadIncsMetadata = 0 - numVgprAddressDbg = self.states.rpga if globalParameters["DebugKernel"] else 0 + numVgprAddressDbg = self.states.rpga if self.debugConfig.debugKernel else 0 #################################### # num vgprs: c write address @@ -3952,7 +3949,7 @@ def readWriteVectors(mat, vw, kernel): self.states.numActivationTypeArgSize = 0 # Will change to 1 if activationType == All self.states.numActivationArgSize = max(1, int(kernel["ProblemType"]["DestDataType"].numRegisters())) self.states.numactivationArgTotalSize = self.states.numActivationArgSize * kernel["ProblemType"]["ActivationType"].getAdditionalArgNum() - self.states.numSgprAddressDbg = self.states.rpga if globalParameters["DebugKernel"] else 0 + self.states.numSgprAddressDbg = self.states.rpga if self.debugConfig.debugKernel else 0 #################################### # num sgprs: global read increments @@ -4024,7 +4021,7 @@ def readWriteVectors(mat, vw, kernel): self.defineSgpr("OrigLoopCounter", 1) - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: self.defineSgpr("AddressDbg", self.states.numSgprAddressDbg) self.defineSgpr("DebugKernelItems", 1) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 38b04a08c5..ed6ebfbefe 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -41,7 +41,7 @@ from .TensilePass import getActivationFunctionModuleName, getActivationBranchModuleName from .TensileInstructions.Containers import HWRegContainer from .Component import Component -from .KernelWriter import KernelWriter, ConstValues, StateValues, StateVgprs, CodeModules +from .KernelWriter import KernelWriter, DebugConfig from .KernelWriterModules import * from .SolutionStructs import isPackedIndex from .AsmStoreState import StoreState, VectorDataTypes @@ -67,8 +67,8 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion) + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig) def getSourceFileString(self, kernel, useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" @@ -740,7 +740,7 @@ def macroAndSet(self, kernel, tPA, tPB) -> Module: module.add(RegSet("v", "vgprSerial", self.states.startVgprSerial)) - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: module.add(RegSet("v", "vgprAddressDbg", \ self.states.startVgprAddressDbg)) #module.addComment0("Occu: %u waves/simd" % self.numWavesPerSimd ) @@ -1225,7 +1225,7 @@ def loadBatchedAddress(self, kernel, Batch, tmpSgprResource: RegisterPoolResourc def getKernelArgLoadModule(self, kernel, sgprStartIdx, numsOfLoad, preloadNum): kernelArgs = Module("load arguments") kernelArgs.addComment1("Load Kernel Args") - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: kernelArgs.add(self.argLoader.loadKernArg("AddressDbg", "KernArgAddress", dword=2)) self.argLoader.resetOffset() kernelArgs.addModuleAsFlatItems(self.argLoader.loadAllKernArg(sgprStartIdx, "KernArgAddress", numsOfLoad, preloadNum)) @@ -1961,7 +1961,7 @@ def calculateWG(): ######################################## # Debug Buffer - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: module.addComment1("Debug Buffer") # nwg0 FIXME use NumWorkGroups0 diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 9082f9e26c..63882fdc39 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -26,6 +26,7 @@ print("This file can no longer be run as a script. Run 'Tensile/bin/Tensile' instead.") exit(1) +import os import subprocess import sys import argparse @@ -61,7 +62,7 @@ def executeStepsInConfig( asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - dbconfig: DebugConfig + debugConfig: DebugConfig ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -87,7 +88,7 @@ def executeStepsInConfig( ############################################################################## if "BenchmarkProblems" in config: BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, \ - cCompiler, outputPath, buildTmpPath, config["ShortNames"], dbconfig) + cCompiler, outputPath, buildTmpPath, config["ShortNames"], debugConfig) print1("") ############################################################################## @@ -306,24 +307,24 @@ def store_max_frequency(max_frequency): return False - def debugConfig(config: dict) -> DebugConfig: - dbConfig = DebugConfig() +def makeDebugConfig(config: dict) -> DebugConfig: + debugConfig = DebugConfig() if "EnableAsserts" in config: - dbConfig.enableAsserts = config["EnableAsserts"] + debugConfig.enableAsserts = config["EnableAsserts"] if "EnableDebugA" in config: - dbConfig.enableDebugA = config["EnableDebugA"] + debugConfig.enableDebugA = config["EnableDebugA"] if "EnableDebugB" in config: - dbConfig.enableDebugB = config["EnableDebugB"] + debugConfig.enableDebugB = config["EnableDebugB"] if "EnableDebugC" in config: - dbConfig.enableDebugC = config["EnableDebugC"] + debugConfig.enableDebugC = config["EnableDebugC"] if "ExpectedValueC" in config: - dbConfig.expectedValueC = config["ExpectedValueC"] + debugConfig.expectedValueC = config["ExpectedValueC"] if "ForceCExpectedValue" in config: - dbConfig.forceCExpectedValue = config["ForceCExpectedValue"] + debugConfig.forceCExpectedValue = config["ForceCExpectedValue"] if "DebugKernel" in config: - dbConfig.debugKernel = config["DebugKernel"] - return dbConfig + debugConfig.debugKernel = config["DebugKernel"] + return debugConfig ################################################################################ @@ -437,11 +438,11 @@ def Tensile(userArgs): srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) overrideParameters = argUpdatedGlobalParameters(args) - + if "ShortNames" not in config: - config["ShortNames"] = args["ShortNames"] - - dbConfig = debugConfig(config) + config["ShortNames"] = args.shortNames + + debugConfig = makeDebugConfig(config) for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) @@ -450,7 +451,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, dbConfig, outputPath, asmToolchain, srcToolchain, cCompiler) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler, debugConfig) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 29f47d4eef..f33e11d2a3 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -590,8 +590,8 @@ def validLogicFile(p: Path): kernels, kernelHelperObjs, kernelWriterAssembly, - compress=arguments["UseCompression"], arguments["ShortNames"], + compress=arguments["UseCompression"], ) archs = [ From 8e8ed08682e9c90a35e11443e6b0b189a5de83d2 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 23:03:09 +0000 Subject: [PATCH 041/105] Remove code from file and force generation globals --- tensilelite/Tensile/Common/GlobalParameters.py | 8 ++++---- tensilelite/Tensile/KernelWriter.py | 11 +++++------ .../Tensile/TensileCreateLibrary/ParseArguments.py | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 0075ac2ec8..11b8e79968 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -57,9 +57,9 @@ True # T=On hip, use the timestamps for kernel start and stop rather than separate events. Can provide more accurate kernel timing. For GlobalSplitU kernels, recommend disabling this to provide consistent ) # timing between GSU / non-GSU kernels -globalParameters["CodeFromFiles"] = ( - True # if False byte arrays will be generated during Benchmarking phase as before -) +#globalParameters["CodeFromFiles"] = ( +# True # if False byte arrays will be generated during Benchmarking phase as before +#) globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't globalParameters["HardwareMonitor"] = ( True # False: disable benchmarking client monitoring clocks using rocm-smi. @@ -130,7 +130,7 @@ # even if error occurs in kernel generation (ie due to resource overflow), # generate the kernel source anyway. Tensile will also attempt to run # the kernel. Useful to examine and debug overflow errors. -globalParameters["ForceGenerateKernel"] = 0 +# globalParameters["ForceGenerateKernel"] = 0 ######################################## # less common diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 39dcc1103e..cfe623dc1a 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -357,6 +357,7 @@ class DebugConfig(NamedTuple): expectedValueC: float=16.0 forceCExpectedValue: bool=False debugKernel: bool=False + forceGenerateKernel: bool=False ################################################################################ # Kernel Writer @@ -3201,7 +3202,7 @@ def readWriteVectors(mat, vw, kernel): # Check if the address setup code for LWA and GRO causes register growth. # This is not an error condition but bears further investigation. # Realistically we just have the GlobalToLocal VGPRs, all else is growth. - self.states.preventVgprOverflowDuringNewTile = 0 and not globalParameters["ForceGenerateKernel"] + self.states.preventVgprOverflowDuringNewTile = 0 and not self.debugConfig.forceGenerateKernel # For Beta: # Rather than waiting for all loads to finish with s_waitcnt vmcnt(0), interleave @@ -5007,7 +5008,7 @@ def _getKernelSource(self, kernel: Solution): fileString += str(kb) if error != 0: - if globalParameters["ForceGenerateKernel"]: + if self.debugConfig.forceGenerateKernel: printWarning("Generating kernel source resulted in error {}, but ForceGenerateKernel=1 so saving source".format(error)) else: raise RuntimeError("Generating kernel source resulted in error {}".format(error)) @@ -5034,8 +5035,7 @@ def getKernelName(self, kernel): def getSourceFileString(self, kernel) -> Tuple[int, str]: """ Returns a string suitable for placing in Kernels.cpp. This means the actual kernel source in the case - of a source kernel, or an assembled code object byte array definition in the case of an assembly kernel, - or an empty string in the case that CodeFromFiles is true. + of a source kernel, or an assembled code object byte array definition in the case of an assembly kernel. In the case of an assembly kernel, this function has the side effect of creating the following files: * An assembly source file @@ -5048,8 +5048,7 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: def getHeaderFileString(self, kernel): kernelName = self.getKernelName(kernel) fileString = "" # CHeader - if not globalParameters["CodeFromFiles"]: - fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName + fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName return fileString diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index e7fd5b6ced..c35b96920b 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -201,7 +201,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: if args.CmakeCxxCompiler: os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler arguments["ShortNames"] = args.ShortNames - arguments["CodeFromFiles"] = False arguments["LogicFormat"] = args.LogicFormat arguments["LibraryFormat"] = args.LibraryFormat if args.no_enumerate: From 59a2bd64db7d8c1a1e429ae292f49d81cdc2feb4 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 23:31:01 +0000 Subject: [PATCH 042/105] Remove HipClangVersion --- tensilelite/Tensile/KernelWriter.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index cfe623dc1a..a3e3e7c438 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -4971,10 +4971,7 @@ def _shortenFileBase(self, kernel): def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): kernelName = self.getKernelFileBase(useShortNames, kernel) with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: - hipccver = globalParameters['HipClangVersion'].split(".") - hipccMaj = int(hipccver[0]) - hipccPatch = int(hipccver[2].split("-")[0]) - if not (hipccMaj >= 6 and hipccPatch >= 32650): + if not (self.amdClangVersion.major >= 6 and self.amdClangVersion.patch >= 32650): code = [] for line in f.readlines(): if "amdhsa_user_sgpr_kernarg_preload" not in line: @@ -4983,8 +4980,8 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): else: code = f.read() - self.tPA = tensorParametersA = {} - self.tPB = tensorParametersB = {} + self.tPA = {} + self.tPB = {} self.states.kernel = kernel self.states.language = "ASM" self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] From 66b5db2fa742d1f60efba5f42176976403d7e8ce Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 11 Feb 2025 23:42:52 +0000 Subject: [PATCH 043/105] remove splitgsu and print rejection reason in kernel writers --- tensilelite/Tensile/Common/GlobalParameters.py | 8 ++++---- tensilelite/Tensile/KernelWriter.py | 2 ++ tensilelite/Tensile/KernelWriterAssembly.py | 6 +++--- tensilelite/Tensile/Tensile.py | 7 +++++++ 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 11b8e79968..aefe7fef28 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -138,9 +138,9 @@ globalParameters["CMakeBuildType"] = ( "Release" # whether benchmark clients and library client should be release or debug ) -globalParameters["PrintSolutionRejectionReason"] = ( - False # when a solution is marked as invalid, print why -) +#globalParameters["PrintSolutionRejectionReason"] = ( +# False # when a solution is marked as invalid, print why +#) globalParameters["LogicFormat"] = "yaml" # set library backend (yaml, or json) globalParameters["LibraryFormat"] = "yaml" # set library backend (yaml, or msgpack) @@ -202,7 +202,7 @@ globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp # debug for assembly -globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM +#globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM # Tensor printing controls: globalParameters["PrintTensorA"] = 0 # Print TensorA after initialization diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index a3e3e7c438..bd1ee9f6c5 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -358,6 +358,8 @@ class DebugConfig(NamedTuple): forceCExpectedValue: bool=False debugKernel: bool=False forceGenerateKernel: bool=False + printSolutionRejectionReason: bool=False + splitGSU: bool=False ################################################################################ # Kernel Writer diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index ed6ebfbefe..130f54530d 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,7 +48,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection, SemanticVersion +from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DataDirection, SemanticVersion from math import ceil, log, floor from copy import deepcopy @@ -1167,7 +1167,7 @@ def checkResources(self, kernel, mkb: KernelBody): else: msg = "unknown" - if globalParameters["PrintSolutionRejectionReason"]: + if self.debugConfig.printSolutionRejectionReason: printWarning("%s overflowed resources. errorCode=%d, msg=\"%s\", vgprs=%u, sgprs=%u" \ % (self.states.kernelName, self.states.overflowedResources, msg, \ self.vgprPool.size(), self.sgprPool.size())) @@ -9181,7 +9181,7 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, useBiasBackup = self.states.useBias betasBackup = betas edgesBackup = edges - gsuLimit = 1 if noGSUBranch or globalParameters["SplitGSU"] else 2 + gsuLimit = 1 if noGSUBranch or self.debugConfig.splitGSU else 2 if gsuLimit > 1: gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") with self.allocTmpSgpr(1) as tmpSgprGSU: diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 63882fdc39..6968c0e69d 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -324,6 +324,13 @@ def makeDebugConfig(config: dict) -> DebugConfig: debugConfig.forceCExpectedValue = config["ForceCExpectedValue"] if "DebugKernel" in config: debugConfig.debugKernel = config["DebugKernel"] + if "ForceGenerateKernel" in config: + debugConfig.forceGenerateKernel = config["ForceGenerateKernel"] + if "PrintSolutionRejectionReason" in config: + debugConfig.printSolutionRejectionReason = config["PrintSolutionRejectionReason"] + if "SplitGSU" in config: + debugConfig.splitGSU = config["SplitGSU"] + return debugConfig From d314777b00b3095b63c4356d7ce4975ca92ddd50 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Wed, 12 Feb 2025 19:34:16 +0000 Subject: [PATCH 044/105] Finish removing splitGSU and print rejection reason --- tensilelite/Tensile/BenchmarkProblems.py | 30 ++++++++-------- tensilelite/Tensile/Common/Types.py | 12 +++++++ tensilelite/Tensile/Contractions.py | 4 +-- tensilelite/Tensile/KernelWriter.py | 17 ++------- tensilelite/Tensile/KernelWriterAssembly.py | 4 +-- tensilelite/Tensile/LibraryIO.py | 8 ++--- tensilelite/Tensile/LibraryLogic.py | 36 +++++++++---------- tensilelite/Tensile/SolutionLibrary.py | 10 +++--- tensilelite/Tensile/SolutionStructs.py | 29 +++++++-------- tensilelite/Tensile/Tensile.py | 2 +- .../Tensile/TensileCreateLibrary/Run.py | 11 +++--- 11 files changed, 83 insertions(+), 80 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 89f51a784d..26f49968aa 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -47,7 +47,7 @@ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR -def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): +def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, debugConfig: DebugConfig): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -59,12 +59,12 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo solution.update(perm) # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, cxxCompiler) + solutionObject = Solution(solution, debugConfig.splitGSU, cxxCompiler) if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) solutions.append(solutionObject) - elif globalParameters["PrintSolutionRejectionReason"]: + elif debugConfig.printSolutionRejectionReason: print1("rejecting solution " + str(solutionObject)) return solutions @@ -76,7 +76,7 @@ def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: s return Solution(config, cxxCompiler) -def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str): +def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str, printSolutionRejectionReason: bool): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: @@ -107,7 +107,7 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar print1("# Added {} to solutions".format(kernelName)) if solution["Valid"]: solutions.append(solution) - elif globalParameters["PrintSolutionRejectionReason"]: + elif printSolutionRejectionReason: print1("rejecting solution " + str(solution)) return solutions @@ -131,7 +131,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ for solution in tqdm(solutions, "Finding unique solutions"): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = Solution.getKeyNoInternalArgs(kernel, debugConfig.splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -152,15 +152,15 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ codeObjectFiles, _= writeSolutionsAndKernels( \ sourcePath, asmToolchain, srcToolchain, \ solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, errorTolerant=True, fromTensile=True, \ + kernelWriterAssembly, debugConfig.splitGSU, errorTolerant=True, fromTensile=True, \ generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], \ useShortNames=useShortNames) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") - newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler) - newLibrary.applyNaming(kernelMinNaming) + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler, debugConfig.splitGSU) + newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) codeObjectFiles = [os.path.relpath(f, sourcePath) \ @@ -285,10 +285,10 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG maxPossibleSolutions = len(forkPermutations) regSolutions = generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler) + benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, debugConfig) kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, srcToolchain.compiler) + not benchmarkStep.customKernelWildcard, srcToolchain.compiler, debugConfig.printSolutionRejectionReason) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions @@ -299,7 +299,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG # handle no valid solutions if len(solutions) == 0: msg = "Your parameters resulted in 0 valid solutions." - if globalParameters["PrintSolutionRejectionReason"]: + if debugConfig.printSolutionRejectionReason: msg += "\nExamine reject and backtrace messages above to see why" \ "and where solutions were rejected." else: @@ -309,7 +309,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG if globalParameters["PrintLevel"] >= 1: for solution in solutions: - print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution))) + print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution, debugConfig.splitGSU))) print2(HR) # write benchmarkFiles @@ -340,8 +340,8 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG for i in range(0, len(solutions)): solution = solutions[i] solution["SolutionIndex"] = i - solution["SolutionNameMin"] = Solution.getNameMin(solution, solutionMinNaming) - solution["KernelNameMin"] = Solution.getNameMin(solution, solutionMinNaming, True) + solution["SolutionNameMin"] = Solution.getNameMin(solution, solutionMinNaming, debugConfig.splitGSU) + solution["KernelNameMin"] = Solution.getNameMin(solution, solutionMinNaming, debugConfig.splitGSU, True) else: solutions = None print1("# Using cached solution data") diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index ddb6900e3d..bd040c4200 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -7,3 +7,15 @@ class SemanticVersion(NamedTuple): major: int minor: int patch: int + +class DebugConfig(NamedTuple): + enableAsserts: bool=False + enableDebugA: bool=False + enableDebugB: bool=False + enableDebugC: bool=False + expectedValueC: float=16.0 + forceCExpectedValue: bool=False + debugKernel: bool=False + forceGenerateKernel: bool=False + printSolutionRejectionReason: bool=False + splitGSU: bool=False diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 73706d5c92..129f378342 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -660,7 +660,7 @@ def FromSolutionStruct(cls, solution, cxxCompiler: str): return cls.FromOriginalState(solution._state, cxxCompiler, solution.srcName) @classmethod - def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): + def FromOriginalState(cls, d, cxxCompiler, splitGSU: bool, srcName = "", deviceInfo=None): rv = cls() @@ -707,7 +707,7 @@ def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, cxxCompiler, srcName) + rv.originalSolution = OriginalSolution(d, splitGSU, cxxCompiler, srcName) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index bd1ee9f6c5..6dd84f9e7f 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,7 +36,7 @@ from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule -from .Common import globalParameters, printWarning, roundUp, print2, printExit, DataDirection, SemanticVersion, \ +from .Common import globalParameters, printWarning, roundUp, print2, DebugConfig, DataDirection, SemanticVersion, \ INDEX_CHARS, MAX_FILENAME_LENGTH import abc @@ -349,18 +349,6 @@ class ExternClasses: biasSumUnroll: Optional[Component.SumUnroll] = None -class DebugConfig(NamedTuple): - enableAsserts: bool=False - enableDebugA: bool=False - enableDebugB: bool=False - enableDebugC: bool=False - expectedValueC: float=16.0 - forceCExpectedValue: bool=False - debugKernel: bool=False - forceGenerateKernel: bool=False - printSolutionRejectionReason: bool=False - splitGSU: bool=False - ################################################################################ # Kernel Writer ################################################################################ @@ -4986,6 +4974,7 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): self.tPB = {} self.states.kernel = kernel self.states.language = "ASM" + # we already do this in the solution ctor self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] if not globalParameters["AsmCaps"][self.states.version]["SupportedISA"]: self.states.version = (9,0,0) @@ -5027,7 +5016,7 @@ def getKernelFileBase(self, useShortNames: bool, kernel): return fileBase def getKernelName(self, kernel): - kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, True) + kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, True, self.debugConfig.splitGSU) return kernelName @abc.abstractmethod diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 130f54530d..d1fb73a6be 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -41,14 +41,14 @@ from .TensilePass import getActivationFunctionModuleName, getActivationBranchModuleName from .TensileInstructions.Containers import HWRegContainer from .Component import Component -from .KernelWriter import KernelWriter, DebugConfig +from .KernelWriter import KernelWriter from .KernelWriterModules import * from .SolutionStructs import isPackedIndex from .AsmStoreState import StoreState, VectorDataTypes from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DataDirection, SemanticVersion +from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, SemanticVersion from math import ceil, log, floor from copy import deepcopy diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index a266a5b4b9..22bcf25a7a 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -192,12 +192,12 @@ def readJson(filename): data = json.loads(f.read()) return data -def parseSolutionsFile(filename, cxxCompiler): +def parseSolutionsFile(filename, cxxCompiler, splitGSU: bool): """Wrapper function to read and parse a solutions file.""" - return parseSolutionsData(read(filename), filename, cxxCompiler) + return parseSolutionsData(read(filename), filename, cxxCompiler, splitGSU) -def parseSolutionsData(data, srcFile, cxxCompiler): +def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool): """Parses problem sizes and solutions from the data of a solutions file.""" if len(data) < 3: printExit("Solution file {} is missing required fields (len = {} < 3" \ @@ -224,7 +224,7 @@ def parseSolutionsData(data, srcFile, cxxCompiler): # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + solutionObject = Solution(solutionState, splitGSU, cxxCompiler, srcFile) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index c8de72ff2a..1eb30a62cb 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -42,7 +42,7 @@ ################################################################################ # Analyze Problem Type ################################################################################ -def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryLogicPath): +def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryLogicPath, splitGSU: bool): print2(HR) print1("# Analyzing: %s" % problemType) @@ -77,14 +77,14 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL solutionIdx = 0 for solution in solutions: print2("# (%u) %s" % (solutionIdx, Solution.getNameMin(solution, \ - solutionMinNaming))) + solutionMinNaming, splitGSU))) solutionIdx += 1 print2(HR) ###################################### # Create Logic Analyzer logicAnalyzer = LogicAnalyzer( problemType, problemSizesList, solutionsList, \ - dataFileNameList, inputParameters) + dataFileNameList, inputParameters, splitGSU) selectionSolutionsIdsList = None selectionSolutions = None @@ -126,9 +126,9 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL for i in range(0, len(logicAnalyzer.solutions)): s = logicAnalyzer.solutions[i] s["SolutionIndex"] = i - s["SolutionNameMin"] = Solution.getNameMin(s, solutionMinNaming) - s["KernelNameMin"] = Solution.getNameMin(s, solutionMinNaming, True) - print1("(%2u) %s : %s" % (i, Solution.getNameMin(s, solutionMinNaming), Solution.getNameFull(s))) + s["SolutionNameMin"] = Solution.getNameMin(s, solutionMinNaming, splitGSU) + s["KernelNameMin"] = Solution.getNameMin(s, solutionMinNaming, splitGSU, True) + print1("(%2u) %s : %s" % (i, Solution.getNameMin(s, solutionMinNaming, splitGSU), Solution.getNameFull(s, splitGSU))) if enableTileSelection: validSelectionSolutions = SolutionSelectionLibrary.analyzeSolutionSelection(problemType, selectionFileNameList, \ @@ -160,8 +160,8 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL (validSolution, validSolutionInfo) = validSelectionSolution selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, True) + validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) @@ -250,10 +250,11 @@ class LogicAnalyzer: # ENTRY: Init ############################################################################## def __init__(self, problemType, problemSizesList, solutionsList, \ - dataFileNameList, inputParameters): + dataFileNameList, inputParameters, splitGSU: bool): # parameters self.parameters = inputParameters + self.splitGSU = splitGSU # problem type self.problemType = problemType @@ -295,7 +296,7 @@ def __init__(self, problemType, problemSizesList, solutionsList, \ self.solutionTiles = [] for solution in self.solutions: self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.flopsPerMac = self.problemType["DataType"].flopsPerMac() @@ -1122,7 +1123,7 @@ def removeSolution(self, removeSolutionIdx): self.solutionTiles = [] for solution in self.solutions: self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.numSolutions = len(self.solutions) @@ -1173,7 +1174,7 @@ def pruneSolutions(self, keepSolutions): self.solutionTiles = [] for solution in self.solutions: self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.numSolutions = len(self.solutions) @@ -1429,8 +1430,7 @@ def indicesToSerial(self, solutionIdx, indices ): return serial - -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str): +def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1473,7 +1473,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1483,7 +1483,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) dataFileName, solutionsFileName, selectionFileName, solutions) ) for problemType in problemTypes: - logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters, libraryLogicPath) + logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters, libraryLogicPath, splitGSU) filename = os.path.join(libraryLogicPath, \ "{}_{}".format(analysisParameters["ScheduleName"], str(problemType))) @@ -1546,7 +1546,7 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path): +def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) + generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 8fedd3396e..835d3badab 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -489,8 +489,8 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler) for s in solutions]) + def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler, splitGSU) for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ @@ -519,14 +519,14 @@ def state(self): rv["version"] = self.version return rv - def applyNaming(self, naming=None): + def applyNaming(self, splitGSU: bool, naming=None): if naming is None: kernels = itertools.chain(s.originalSolution.getKernels() for s in self.solutions.values()) naming = OriginalSolution.getMinNaming(kernels) for s in list(self.solutions.values()): - s.name = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming) - s.kernelName = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, True) + s.name = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU) + s.kernelName = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU, True) def remapSolutionIndicesStartingFrom(self, curIndex): reIndexMap = {} diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 8e9686cc71..0d27a07598 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -49,7 +49,7 @@ from collections.abc import Mapping from enum import Enum from functools import lru_cache -from typing import List +from typing import List, NamedTuple import collections import math @@ -58,11 +58,11 @@ ######################################## # Print a reject message : -def reject(state, *args): +def reject(state, printSolutionRejectionReason: bool, *args): if state and "NoReject" in state and state["NoReject"]: return - if globalParameters["PrintSolutionRejectionReason"]: + if printSolutionRejectionReason: sys.stdout.write("\nreject: ") for a in args: print(a) @@ -1053,10 +1053,11 @@ def isExtractableIndex(ks, index, tc='x'): class Solution(collections.abc.Mapping): ######################################## - def __init__(self, config, cxxCompiler: str, srcName: str = ""): + def __init__(self, config, splitGSU: bool, cxxCompiler: str, srcName: str = ""): self._name = None self.cxxCompiler = cxxCompiler self.srcName = srcName + self.splitGSU = splitGSU config = config self._state = {} @@ -1116,7 +1117,7 @@ def __init__(self, config, cxxCompiler: str, srcName: str = ""): while True: state = deepcopy(self._state) state["DepthU"] = depthuList[index[0]] - Solution.assignDerivedParameters(state, index, depthuList) + Solution.assignDerivedParameters(state, index, depthuList, splitGSU) if state["Valid"]: self._state = state break @@ -2130,7 +2131,7 @@ def getDivisorName(state, tC): ######################################## # assign all derived parameters @staticmethod - def assignDerivedParameters(state, index, depthuList): + def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -2162,7 +2163,7 @@ def assignDerivedParameters(state, index, depthuList): elif state["GlobalSplitUAlgorithm"] == 'MultipleBuffer': state["_GlobalAccumulation"] = 'MultipleBuffer' elif state["GlobalSplitUAlgorithm"] == 'MultipleBufferSingleKernel': - if (not globalParameters["SplitGSU"]): + if (not splitGSU): state["_GlobalAccumulation"] = 'MultipleBufferSingleKernel' else: if state["GlobalSplitU"] > 1: @@ -4256,12 +4257,12 @@ def getMinNaming(objs): ######################################## @ staticmethod - def getKeyNoInternalArgs(state): + def getKeyNoInternalArgs(state, splitGSU: bool): state_copy = deepcopy(state) state_copy["ProblemType"]["GroupedGemm"] = False - if globalParameters["SplitGSU"]: + if splitGSU: state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] elif state["GlobalSplitU"] > 0: state_copy["GlobalSplitU"] = "M" @@ -4277,7 +4278,7 @@ def getKeyNoInternalArgs(state): return state_copy @ staticmethod - def getNameFull(state): + def getNameFull(state, splitGSU: bool): requiredParameters = {} for key in state: if key in list(validParameters.keys()): @@ -4286,12 +4287,12 @@ def getNameFull(state): # Use MIWaveGroup and MIWaveTile instead of WG and MT requiredParameters["MIWaveTile"] = True requiredParameters["ThreadTile"] = False - return Solution.getNameMin(state, requiredParameters) + return Solution.getNameMin(state, requiredParameters, splitGSU) ######################################## # Get Name Min @ staticmethod - def getNameMin(state, requiredParameters, ignoreInternalArgs = False): + def getNameMin(state, requiredParameters, splitGSU: bool, ignoreInternalArgs = False): if isCustomKernelConfig(state): return state["CustomKernelName"] @@ -4319,7 +4320,7 @@ def getNameMin(state, requiredParameters, ignoreInternalArgs = False): backup = state["GlobalSplitU"] if ignoreInternalArgs: - if globalParameters["SplitGSU"]: + if splitGSU: state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] elif state["GlobalSplitU"] > 0: requiredParameters["GlobalSplitU"] = False @@ -4491,7 +4492,7 @@ def __setitem__(self, key, value): def __str__(self): if self._name is None: - self._name = Solution.getNameFull(self._state) + self._name = Solution.getNameFull(self._state, self.splitGSU) return self._name def __repr__(self): diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 6968c0e69d..b0dee9589d 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -105,7 +105,7 @@ def executeStepsInConfig( libraryLogicConfig = config["LibraryLogic"] else: libraryLogicConfig = {} - LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath) + LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath, debugConfig) print1("") else: print1("# LibraryLogic already done.") diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index f33e11d2a3..3cdc82ba30 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -98,7 +98,7 @@ def processKernelSource(kernelWriterAssembly, ti, useShortNames, kernel) -> Kern ) -def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, globalParameters): +def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, printLevel: bool, splitGSU: bool): removeKernels = [] removeKernelNames = [] removeSolutions = [] @@ -116,7 +116,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, ) print(kernels[kernIdx]["SolutionNameMin"]) removeKernels.append(kernels[kernIdx]) - kName = Solution.getKeyNoInternalArgs(kernels[kernIdx]) + kName = Solution.getKeyNoInternalArgs(kernels[kernIdx], splitGSU) if kName not in removeKernelNames: removeKernelNames.append(kName) removeResults.append(results[kernIdx]) @@ -129,12 +129,12 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, for solution in ( tqdm(solutions, "Finding invalid solutions") - if globalParameters["PrintLevel"] > 1 + if printLevel > 1 else solutions ): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = Solution.getKeyNoInternalArgs(kernel, splitGSU) if kName in removeKernelNames: removeSolutions.append(solution) break @@ -197,6 +197,7 @@ def writeSolutionsAndKernels( kernels, kernelHelperObjs, kernelWriterAssembly, + splitGSU: bool, errorTolerant=False, generateSourcesAndExit=False, compress=True, @@ -237,7 +238,7 @@ def writeSolutionsAndKernels( ) asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels") removeInvalidSolutionsAndKernels( - asmResults, asmKernels, solutions, errorTolerant, globalParameters + asmResults, asmKernels, solutions, errorTolerant, globalParameters["PrintLevel"], splitGSU ) def assemble(ret): From 549f5cb8ab4108a8fc37325fff2beeca0b76a42c Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 11 Feb 2025 17:55:09 -0600 Subject: [PATCH 045/105] feat: move capabilities to object --- tensilelite/Tensile/BenchmarkProblems.py | 57 +++- tensilelite/Tensile/ClientWriter.py | 7 +- .../Tensile/Common/GlobalParameters.py | 80 +++--- tensilelite/Tensile/Common/Types.py | 8 + tensilelite/Tensile/Common/Utilities.py | 9 +- tensilelite/Tensile/Contractions.py | 12 +- tensilelite/Tensile/GenerateSummations.py | 4 +- tensilelite/Tensile/KernelWriter.py | 15 +- tensilelite/Tensile/KernelWriterAssembly.py | 16 +- tensilelite/Tensile/KernelWriterConversion.py | 10 +- tensilelite/Tensile/LibraryIO.py | 26 +- tensilelite/Tensile/LibraryLogic.py | 11 +- tensilelite/Tensile/SolutionLibrary.py | 11 +- tensilelite/Tensile/SolutionStructs.py | 272 +++--------------- tensilelite/Tensile/Tensile.py | 21 +- .../Tensile/TensileCreateLibrary/Run.py | 27 +- .../Tensile/TensileInstructions/Base.py | 12 +- .../TensileLogic/ValidMatrixInstruction.py | 15 +- tensilelite/Tensile/Toolchain/Source.py | 7 +- 19 files changed, 257 insertions(+), 363 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 350ab1fa56..91acf5a29b 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -30,6 +30,7 @@ from copy import deepcopy from pathlib import Path +from typing import Dict from . import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations @@ -41,12 +42,12 @@ from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain -from .Common import globalParameters, HR, print1, print2, \ +from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR -def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): +def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, isaInfoMap: Dict[str, IsaInfo]): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -58,7 +59,7 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo solution.update(perm) # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, cxxCompiler) + solutionObject = Solution(solution, cxxCompiler, isaInfoMap) if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) @@ -69,10 +70,16 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo return solutions -def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, directory=CUSTOM_KERNEL_PATH): +def getCustomKernelSolutionObj( + kernelName, + internalSupportParams, + cxxCompiler: str, + isaInfoMap: Dict[str, IsaInfo], + directory=CUSTOM_KERNEL_PATH + ): """Creates the Solution object for a custom kernel""" config = getCustomKernelConfig(kernelName, internalSupportParams, directory) - return Solution(config, cxxCompiler) + return Solution(config, cxxCompiler, isaInfoMap) def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str): @@ -113,7 +120,7 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path): + asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, isaInfoMap: Dict[str, IsaInfo]): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -142,20 +149,22 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion) + kernelWriterAssembly = KernelWriterAssembly( + kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, isaInfoMap + ) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ sourcePath, asmToolchain, srcToolchain, \ solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, errorTolerant=True, fromTensile=True, \ + kernelWriterAssembly, isaInfoMap, errorTolerant=True, fromTensile=True, \ generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"]) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") - newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler) + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler, isaInfoMap) newLibrary.applyNaming(kernelMinNaming) LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) @@ -200,7 +209,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path + buildTmpPath: Path, benchmarkProblemsPath: Path, isaInfoMap: Dict[str, IsaInfo] ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -281,7 +290,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG maxPossibleSolutions = len(forkPermutations) regSolutions = generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler) + benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, isaInfoMap) kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ not benchmarkStep.customKernelWildcard, srcToolchain.compiler) @@ -314,7 +323,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath) + sourcePath, isaInfoMap) # ^ this mutates solutions # write cache data @@ -379,7 +388,16 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG return (resultsFileBaseFinal, benchmarkTestFails) -def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path): +def main( + config, + useCache, + asmToolchain: AssemblyToolchain, + srcToolchain: SourceToolchain, + isaInfoMap: Dict[str, IsaInfo], + cCompiler: str, + outputPath: Path, + buildTmpPath: Path + ): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) @@ -418,7 +436,18 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # benchmark problem size group benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath) + benchmarkProblemType( + problemTypeConfig, + sizeGroupConfig, + idx, + useCache, + asmToolchain, + srcToolchain, + cCompiler, + buildTmpPath, + benchmarkProblemsPath, + isaInfoMap + ) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 5d3e31367d..6555f78de2 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -30,11 +30,12 @@ from pathlib import Path from enum import Enum from glob import glob +from typing import Dict from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO -from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, \ +from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, IsaInfo, \ LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles @@ -79,7 +80,7 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): +def main(config, cxxCompiler: str, cCompiler: str, isaInfoMap: Dict[str, IsaInfo], outputPath: Path): libraryLogicPath = ensurePath(outputPath / LIBRARY_LOGIC_DIR) clientLibraryPath = ensurePath(outputPath / LIBRARY_CLIENT_DIR) @@ -105,7 +106,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): clientParametersPaths = [] for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary) \ - = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) + = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler, isaInfoMap) functions.append((scheduleName, problemType)) functionNames.append("tensile_%s" % (problemType)) problemSizes = ProblemSizesMock(exactLogic) if exactLogic else ProblemSizesMockDummy() diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index eda9f6ea44..0f95b6249a 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -33,7 +33,8 @@ from Tensile import __version__ from .Architectures import gfxToIsa, isaToGfx -from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps +from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps, initRegisterCaps +from .Types import IsaInfo from .Utilities import locateExe, versionIsCompatible startTime = time.time() @@ -1571,41 +1572,41 @@ def restoreDefaultGlobalParameters(): globalParameters[key] = value -def printTable(rows): - rows = list([[str(cell) for cell in row] for row in rows]) - colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) +# def printTable(rows): +# rows = list([[str(cell) for cell in row] for row in rows]) +# colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - for row in rows: - for width, cell in zip(colWidths, row): - pad = " " * (width - len(cell)) - print(pad, cell, sep="", end=" ") - print() +# for row in rows: +# for width, cell in zip(colWidths, row): +# pad = " " * (width - len(cell)) +# print(pad, cell, sep="", end=" ") +# print() -def printCapTable(parameters): - import itertools +# def printCapTable(parameters, isaInfoMap): +# import itertools - archs = [(0, 0, 0)] + parameters["SupportedISA"] - gfxNames = list(map(isaToGfx, archs)) +# archs = [(0, 0, 0)] + parameters["SupportedISA"] +# gfxNames = list(map(isaToGfx, archs)) - headerRow = ["cap"] + gfxNames +# headerRow = ["cap"] + gfxNames - def capRow(caps, cap): - return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] +# def capRow(caps, cap): +# return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] - allAsmCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) - ) - allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) - asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] +# allAsmCaps = set( +# itertools.chain(*[caps.keys() for arch, caps in isaInfoMap.items()]) +# ) +# allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) +# asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] - allArchCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) - ) - allArchCaps = sorted(allArchCaps) - archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] +# allArchCaps = set( +# itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) +# ) +# allArchCaps = sorted(allArchCaps) +# archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] - printTable([headerRow] + asmCapRows + archCapRows) +# printTable([headerRow] + asmCapRows + archCapRows) def assignGlobalParameters(config, cxxCompiler=None): @@ -1703,23 +1704,22 @@ def assignGlobalParameters(config, cxxCompiler=None): globalParameters["CurrentISA"] = (9, 0, 6) printWarning("Failed to detect ISA so forcing (gfx906) on windows") - globalParameters["AsmCaps"] = {} - globalParameters["ArchCaps"] = {} - globalParameters["AsmBugs"] = {} - + isaInfoMap = {} for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: - globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) - globalParameters["ArchCaps"][v] = initArchCaps(v) - globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + asmCaps = initAsmCaps(v, cxxCompiler, False) + archCaps = initArchCaps(v) + regCaps = initRegisterCaps(v, archCaps) + asmBugs = initAsmBugs(asmCaps) + isaInfoMap[v] = IsaInfo(cxxCompiler, asmCaps, archCaps, regCaps, asmBugs) - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) + # if globalParameters["PrintLevel"] >= 1: + # printCapTable(globalParameters, isaInfoMap) globalParameters["SupportedISA"] = list( [ - i - for i in globalParameters["SupportedISA"] - if globalParameters["AsmCaps"][i]["SupportedISA"] + v + for v in globalParameters["SupportedISA"] + if isaInfoMap[v].asmCaps["SupportedISA"] ] ) @@ -1770,6 +1770,8 @@ def assignGlobalParameters(config, cxxCompiler=None): printWarning("Global parameter %s = %s unrecognised." % (key, value)) globalParameters[key] = value + return isaInfoMap + def setupRestoreClocks(): import atexit diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index ddb6900e3d..ad415caeb8 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -1,7 +1,15 @@ +from dataclasses import dataclass from typing import NamedTuple, Tuple IsaVersion = Tuple[int, int, int] +@dataclass +class IsaInfo: + assemblerPath: str + asmCaps: dict + archCaps: dict + regCaps: dict + asmBugs: dict class SemanticVersion(NamedTuple): major: int diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 291ad52c28..b39875d1a9 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -5,11 +5,12 @@ import sys import time from enum import Enum -from typing import List, Tuple +from typing import List, Tuple, Dict from Tensile import __version__ from .Architectures import isaToGfx +from .Types import IsaInfo # get param values from structures. @@ -42,7 +43,7 @@ def locateExe(defaultPath, exeName): # /opt/rocm/bin, hip-clang return None -def splitArchs(params: dict, fromTensile=False) -> Tuple[List[str], List[str]]: +def splitArchs(params: dict, isaInfoMap: Dict[str, IsaInfo], fromTensile=False) -> Tuple[List[str], List[str]]: """ Splits and processes the architecture strings based on the provided parameters. @@ -58,7 +59,7 @@ def splitArchs(params: dict, fromTensile=False) -> Tuple[List[str], List[str]]: def isSupported(arch): return ( - params["AsmCaps"][arch]["SupportedISA"] and params["AsmCaps"][arch]["SupportedSource"] + isaInfoMap[arch].asmCaps["SupportedISA"] and isaInfoMap[arch].asmCaps["SupportedSource"] ) if ";" in params["Architecture"]: @@ -103,7 +104,7 @@ def ensurePath(path): except FileExistsError: pass except OSError: - printExit('Failed to create directory "%s" ' % (path)) + raise OSError('Failed to create directory "%s" ' % (path)) return path diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 73706d5c92..1012b5b752 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -22,13 +22,15 @@ # ################################################################################ +from typing import Dict + from .Activation import ActivationType from .TensileInstructions import DataType from . import Hardware from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering +from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering, IsaInfo @state_key_ordering class FreeIndex: @@ -656,11 +658,11 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, cxxCompiler: str): - return cls.FromOriginalState(solution._state, cxxCompiler, solution.srcName) + def FromSolutionStruct(cls, solution, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo]): + return cls.FromOriginalState(solution._state, cxxCompiler, isaInfoMap, solution.srcName) @classmethod - def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): + def FromOriginalState(cls, d, cxxCompiler, isaInfoMap, srcName = "", deviceInfo=None): rv = cls() @@ -707,7 +709,7 @@ def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, cxxCompiler, srcName) + rv.originalSolution = OriginalSolution(d, cxxCompiler, isaInfoMap, srcName) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index 3b791d1c92..84fc1afd3c 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -64,7 +64,7 @@ def GenerateSummations(userArgs): inputLogicPath = userArgs[0] outputPath = userArgs[1] - assignGlobalParameters({}) + isaInfoMap = assignGlobalParameters({}) cxxCompiler, cCompiler = validateToolchain(ToolchainDefaults.CXX_COMPILER, ToolchainDefaults.C_COMPILER) currentISA = globalParameters["CurrentISA"] @@ -93,7 +93,7 @@ def GenerateSummations(userArgs): # same as the initial logic with the summation model added. To preseve the original # logic we also read in the raw unaltered version of the logic and stage the content # to write the final logic. - logic = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) + logic = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler, isaInfoMap) rawLogic = LibraryIO.rawLibraryLogic(logicFileName) # If we cannot read the logic file then skip it diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index a544e68aa6..05c8d4d125 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -37,7 +37,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import globalParameters, printWarning, roundUp, print2, printExit, DataDirection, SemanticVersion, \ - INDEX_CHARS, MAX_FILENAME_LENGTH + INDEX_CHARS, MAX_FILENAME_LENGTH, IsaInfo import abc import os @@ -357,12 +357,20 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): + def __init__( + self, + kernelMinNaming, + kernelSerialNaming, + assembler: str, + amdClangVersion: SemanticVersion, + isaInfoMap: Dict[str, IsaInfo] + ): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler self.amdClangVersion = amdClangVersion self.ti = None + self.isaInfoMap = isaInfoMap self.do = {} self.do["PreLoop"] = True @@ -2890,6 +2898,7 @@ def initKernel(self, kernel, tensorParametersA, tensorParametersB): self.ti = TensileInstructions() self.ti.init(version, self.assembler) self.ti.setKernelInfo(version, kernel["WavefrontSize"]) + self.ti.getArchCaps self.consts = ConstValues() self.states = StateValues(version=version, kernel=kernel, kernelName=self.getKernelName(kernel)) @@ -4984,7 +4993,7 @@ def _getCustomKernelSource(self, kernel, CustomKernelDirectory): self.states.kernel = kernel self.states.language = "ASM" self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] - if not globalParameters["AsmCaps"][self.states.version]["SupportedISA"]: + if self.isaInfoMap[self.states.version].asmCaps["SupportedISA"]: self.states.version = (9,0,0) printWarning(f"ISA: {self.version} is not supported; overriding with {self.states.version}") diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 6ecfb82278..1fc5947189 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,12 +48,13 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection, SemanticVersion +from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, \ + INDEX_CHARS, DataDirection, SemanticVersion, IsaInfo from math import ceil, log, floor from copy import deepcopy from dataclasses import dataclass, field -from typing import NamedTuple, Tuple +from typing import NamedTuple, Tuple, Dict import os import subprocess @@ -67,8 +68,15 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion) + def __init__( + self, + kernelMinNaming, + kernelSerialNaming, + assembler: str, + isaInfoMap: Dict[str, IsaInfo], + amdClangVersion: SemanticVersion + ): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, isaInfoMap) def getSourceFileString(self, kernel) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 582205c7cf..6e5312bbb6 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -22,16 +22,17 @@ # ################################################################################ +from typing import Dict from copy import deepcopy from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, gfxToIsa, isaToGfx, INDEX_CHARS +from .Common import globalParameters, IsaInfo, gfxToIsa, isaToGfx, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): - def __init__(self, state, load_vw): + def __init__(self, state, load_vw, isaInfoMap: Dict[str, IsaInfo]): super().__init__() self.state["ProblemType"] = deepcopy(state["ProblemType"]) @@ -65,6 +66,7 @@ def __init__(self, state, load_vw): # derive parameter self.language = "HIP" self.kernelName = self.getKernelName() + self.isaInfoMap = isaInfoMap self.datatype = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) self.int32Str = DataType('int32').toDevice(self.language) if self.state["ProblemType"]["DataType"].isInt8() and self.state["ProblemType"]["ComputeDataType"].isSingle() and self.state["ProblemType"]["HighPrecisionAccumulate"]: @@ -533,8 +535,8 @@ def kernelBody(self): kStr += " float2 accumVec2(accum[2], accum[3]);" + self.endLine canPKF32Arch = [] for arch in self.supportedArchs: - archTuple = tuple(arch) - if globalParameters["AsmCaps"][archTuple]['v_pk_add_f32']: + isa = tuple(arch) + if self.isaInfoMap[isa].asmCaps['v_pk_add_f32']: canPKF32Arch.append(arch) defineStr = [] if len(canPKF32Arch) > 0: diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index a266a5b4b9..c684ebe96d 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -26,9 +26,9 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__ +from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__, print1, IsaInfo -from typing import NamedTuple, List +from typing import NamedTuple, List, Dict import os import sys @@ -192,12 +192,12 @@ def readJson(filename): data = json.loads(f.read()) return data -def parseSolutionsFile(filename, cxxCompiler): +def parseSolutionsFile(filename, cxxCompiler, isaInfoMap): """Wrapper function to read and parse a solutions file.""" - return parseSolutionsData(read(filename), filename, cxxCompiler) + return parseSolutionsData(read(filename), filename, cxxCompiler, isaInfoMap) -def parseSolutionsData(data, srcFile, cxxCompiler): +def parseSolutionsData(data, srcFile, cxxCompiler, isaInfoMap): """Parses problem sizes and solutions from the data of a solutions file.""" if len(data) < 3: printExit("Solution file {} is missing required fields (len = {} < 3" \ @@ -224,7 +224,7 @@ def parseSolutionsData(data, srcFile, cxxCompiler): # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + solutionObject = Solution(solutionState, cxxCompiler, isaInfoMap, srcFile) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) @@ -240,12 +240,12 @@ class LibraryLogic(NamedTuple): exactLogic: list library: SolutionLibrary.MasterSolutionLibrary -def parseLibraryLogicFile(filename, cxxCompiler, archs=None): +def parseLibraryLogicFile(filename, cxxCompiler, isaInfoMap: Dict[str, IsaInfo], archs=None): """Wrapper function to read and parse a library logic file.""" - return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, archs) + return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, isaInfoMap, archs) -def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): +def parseLibraryLogicData(data, srcFile, cxxCompiler, isaInfoMap: Dict[str, IsaInfo], archs=None): """Parses the data of a library logic file.""" if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) @@ -271,7 +271,7 @@ def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): problemType = ProblemType(data["ProblemType"]) # unpack solution - def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: + def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: if solutionState["KernelLanguage"] == "Assembly": solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) else: @@ -290,7 +290,7 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + solutionObject = Solution(solutionState, cxxCompiler, isaInfoMap, srcFile) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -302,9 +302,9 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: printExit(f"ProblemType in library logic file {srcFile} doesn't match solution(idx={solIdx}): \n{results}") return solutionObject - solutions = [solutionStateToSolution(solutionState, cxxCompiler) for solutionState in data["Solutions"]] + solutions = [solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) for solutionState in data["Solutions"]] - newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, cxxCompiler) + newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, cxxCompiler, isaInfoMap) return LibraryLogic(data["ScheduleName"], data["ArchitectureName"], problemType, solutions, \ data.get("ExactLogic"), newLibrary) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index c8de72ff2a..ec233e9d21 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -23,8 +23,9 @@ ################################################################################ from pathlib import Path +from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ - assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ + assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, IsaInfo, \ LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR from .SolutionStructs import Solution from . import LibraryIO @@ -1430,7 +1431,7 @@ def indicesToSerial(self, solutionIdx, indices ): -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str): +def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo]): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1473,7 +1474,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, isaInfoMap) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1546,7 +1547,7 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path): +def main(config, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo], outputPath: Path): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) + generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, isaInfoMap) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 8fedd3396e..8f808c3387 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -23,13 +23,14 @@ ################################################################################ import itertools +from typing import Dict from . import Properties from . import Hardware from . import Common from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state +from .Common import state, IsaInfo class SingleSolutionLibrary: Tag = "Single" @@ -301,6 +302,7 @@ def FromOriginalState(cls, origData, origSolutions, cxxCompiler, + isaInfoMap: Dict[str, IsaInfo], solutionClass=Contractions.Solution, libraryOrder=None, placeholderName='TensileLibrary'): @@ -461,6 +463,7 @@ def selection(d, problemType, solutions, library, placeholderName): MasterSolutionLibrary.FromOriginalState(origData, origSolutions, cxxCompiler, + isaInfoMap, solutionClass, libraryOrder[placeholderIndex:], placeholderName) @@ -468,7 +471,7 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, cxxCompiler) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct(s, cxxCompiler, isaInfoMap) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -489,8 +492,8 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler) for s in solutions]) + def BenchmarkingLibrary(cls, solutions, cxxCompiler, isaInfoMap): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler, isaInfoMap) for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index c374c13932..4d34ab9bbd 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -37,19 +37,19 @@ from .CustomKernels import isCustomKernelConfig -from .Common import assignParameterWithDefault, \ +from .Common import assignParameterWithDefault, IsaInfo, \ defaultProblemType, defaultSolution, \ defaultInternalSupportParams, \ globalParameters, internalParameters, \ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS + validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS, print1 from collections import OrderedDict from collections.abc import Mapping from enum import Enum from functools import lru_cache -from typing import List +from typing import List, Dict import collections import math @@ -1053,9 +1053,10 @@ def isExtractableIndex(ks, index, tc='x'): class Solution(collections.abc.Mapping): ######################################## - def __init__(self, config, cxxCompiler: str, srcName: str = ""): + def __init__(self, config, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo], srcName: str = ""): self._name = None self.cxxCompiler = cxxCompiler + self.isaInfoMap = isaInfoMap self.srcName = srcName config = config @@ -1079,7 +1080,8 @@ def __init__(self, config, cxxCompiler: str, srcName: str = ""): assignParameterWithDefault(self._state, key, config, defaultSolution) if 'ISA' not in self._state: if 'ISA' in config: - if not globalParameters["AsmCaps"][tuple(config['ISA'])]["SupportedISA"]: + isa = tuple(config['ISA']) + if not self.isaInfoMap[isa].asmCaps["SupportedISA"]: defaultIsa = [9,0,0] print("warning: ISA:", config['ISA'], " is not supported; overriding with ", defaultIsa) self._state['ISA'] = defaultIsa @@ -1116,7 +1118,7 @@ def __init__(self, config, cxxCompiler: str, srcName: str = ""): while True: state = deepcopy(self._state) state["DepthU"] = depthuList[index[0]] - Solution.assignDerivedParameters(state, index, depthuList) + Solution.assignDerivedParameters(state, index, depthuList, isaInfoMap) if state["Valid"]: self._state = state break @@ -1204,7 +1206,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, self.isaInfoMap)) for btype in typeList: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1217,7 +1219,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, self.isaInfoMap)) else: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1228,7 +1230,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, self.isaInfoMap)) def initActivationEnumHeaderObjects(self): self.activationEnumHeaderObjects = [] @@ -1293,19 +1295,19 @@ def getKernelConversionObjects(self): return self.conversionKernelObjects @staticmethod - def getMIOutputInfo(state): + def getMIOutputInfo(state, isaInfoMap: Dict[str, IsaInfo]): outputVectorWidth = 4 RegsPerOut = 1 isa = tuple(state["ISA"]) - if globalParameters["AsmCaps"][isa]['HasMFMA']: + if isaInfoMap[isa].asmCaps['HasMFMA']: if state["ProblemType"]["DataType"].MIOutputTypeNameAbbrev() == 'f64': outputVectorWidth, RegsPerOut = 1, 2 else: outputVectorWidth, RegsPerOut = 4, 1 - elif globalParameters["AsmCaps"][isa]['HasWMMA_V1']: + elif isaInfoMap[isa].asmCaps['HasWMMA_V1']: outputVectorWidth, RegsPerOut = 1, 1 - elif globalParameters["AsmCaps"][isa]['HasWMMA_V2']: + elif isaInfoMap[isa].asmCaps['HasWMMA_V2']: outputVectorWidth, RegsPerOut = 8, 1 else: print("WARNING: unexpect code flow") @@ -1315,7 +1317,7 @@ def getMIOutputInfo(state): ######################################## # assign tile sizes @staticmethod - def assignProblemIndependentDerivedParameters(state): + def assignProblemIndependentDerivedParameters(state, isaInfoMap: Dict[str, IsaInfo]): if globalParameters["NewClient"] != 2: print("WARNING: Old client deprecated, NewClient parameter being set to 2.") @@ -1334,7 +1336,7 @@ def assignProblemIndependentDerivedParameters(state): if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): reject(state, "General Batched GEMM only support GEMM OperationType") - Solution.MatrixInstructionToMIParameters(state) + Solution.MatrixInstructionToMIParameters(state, isaInfoMap) EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None if EnableMatrixInstruction == None: if ("MIBlock" in state and len(state["MIBlock"]) == 6) \ @@ -1356,7 +1358,7 @@ def assignProblemIndependentDerivedParameters(state): state["MatrixInstBN"] = state["MIBlock"][5] state["LocalSplitU"] = 1 - state["MIOutputVectorWidth"], state["MIRegPerOut"] = Solution.getMIOutputInfo(state) + state["MIOutputVectorWidth"], state["MIRegPerOut"] = Solution.getMIOutputInfo(state, isaInfoMap) if state["MatrixInstM"] == 4: state["ThreadTile0"] = state["MIWaveTile"][0] * state["MIOutputVectorWidth"] @@ -1534,194 +1536,10 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota state["LSP%s"%tc] = state["NumThreads"] // state["WavefrontSize"] return True - - - ######################################## - # Sets the Global Read Tile dims (para, perp) - # This information controls which threads read which addresses from global mem) - # Output from this function: - # state[NumLoadsCoalesced*] - # state[NumLoadsPerpendicular*] - # state[LSC*] - # state[LSP*] - # state[GlobalReadVectorWidth] - # - # LSC and LSP define the shape of the PerLoadTile, measured in elements. - # LSC*LSP is the elements loaded by a single instruction across all - # threads in the group. - # LSC is the number of elements loaded in the para(coalesced) dimension - # LSP is the number of elements loaded in the perp(noncoalesced) dimension - # PerLoadTile is always rectangular. - # When BufferLoad=1, the area (LSC*LSP) can be larger than NumThreads. - # In this case, some threads will generate a dummy OOB GRO. - # Related fields: - # LVC = LSC/GRVW (LVCA = LSCA/GLVWA) - # LVP = LSP/GRVW (LVPA = LSPA/GLVWA) - # - # NumLoadsCoalesced and NumLoadsPerpendicular define the number of times the - # PerLoadTile is loaded in each dimension to fetch the LoadTile - # LoadTile = (LSC * NumLoadsCoalesced) * (LSP * NumLoadsPerpendicular). - # For Fractional, the LoadTile can be larger than the MacroTile. Buffer - # loads will clip any OOB references to 0 and will also avoid writing these - # into LDS. - - # Fractional load algorithm: - # - Each load instruction loads one or more (complete) rows of the load tile. - # - Each row is LSC elements wide - # - Rows are complete and do not wrap. This allows a single base GRO VGPR - # to be used for all loads in the tile. - # - Some work-items in the load may not perform useful work. These WI will - # set their GRO to a large OOB number so as to do no harm - # - Some G2L registers space may be unused as well. - # - The 'used' message at the bottom of this routine computes and prints the - # wasted register space. - # - The wasted space is removed when the data is written to LDS- the LWO - # for work-items beyond the valid ones are set to safely write to OOB locations. - - # - In cases where each load is loading multiple rows (multiple lines of lsc - # elements), the last load is allowed to load fewer lines than the others. - # The KernelWriterAssembly will modify the LWO for the last load. This allows - # flexibility in the unroll factors for example. - @staticmethod - def setGlobalLoadTileDimFractional(state, tc, depthU): - - assert(depthU > 0) - dbFract = 0 - - # parDim, perpDim define the LoadTile and are measured in elements - if state["ProblemType"]["TLU%s"%tc]: - parDim = state["MacroTile%s"%tc] - perpDim = depthU - else: - parDim = depthU - perpDim = state["MacroTile%s"%tc] - - if dbFract: - print("\ninfo: %s Fractional MT%u_%u_%u Par=%u Perp=%u WG%02u_%02u_%02u NumThreads=%u GRWV%s=%u" \ - % (tc, state["MacroTile0"], state["MacroTile1"], depthU, \ - parDim, perpDim, \ - state["WorkGroup"][0], state["WorkGroup"][1], state["LocalSplitU"], \ - state["NumThreads"], tc, state["GlobalReadVectorWidth%s"%tc])) - - # Try to find a GRVW which is smaller than the LSC and also does not force - # the LSC to wrap - both of these conditions can be tested with lsc % grvw ==0. - # Each iteration divides GRWV by 2 which provides finer granularity - # and a possible opportunity to handle the lsc - grvw = state["GlobalReadVectorWidth%s"%tc] - minGrvw = 2 if state["ProblemType"]["DataType"].isHalf() and \ - globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 - # TODO- check this for int8 and fractional load - # minGrvw = 4 if state["ProblemType"]["DataType"].isInt8() and \ - # globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 - bestVw = -1 - while grvw >= minGrvw: - # Per instruction across the entire group: - elementsLoadedPerInst = state["NumThreads"]*grvw - mik = 1 - if (state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]): - mik = state["MatrixInstK"] * state["LocalSplitU"] // state["MIInputPerThread"] - elementsLoadedPerInst //= mik - # LSC, LSP - #elements loaded along specified dim with each load - if parDim >= elementsLoadedPerInst: - # entire work-group can work on (part) of the same row - state["LSC%s"%tc] = elementsLoadedPerInst - state["LSP%s"%tc] = mik if state["ProblemType"]["TLU%s"%tc] else state["MatrixInstK"] - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim , state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = 1 - else: - # work-group exceeds read dimension so wraps to multiple rows - state["LSC%s"%tc] = parDim - state["LSP%s"%tc] = min(perpDim, elementsLoadedPerInst // parDim) - state["NumLoadsCoalesced%s"%tc] = 1 - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - # Vector loads can't wrap to next P dim, so LSC must be divisible by vector elements; - if dbFract: - print(" lsc search : lsc(%u) %% grvw(%u) = %u (?0)" % (state["LSC%s"%tc], grvw, state["LSC%s"%tc] % grvw)) - if state["LSC%s"%tc] % grvw == 0: - bestVw = grvw - # Try to shrink GRVW if possible while keeping same LSC and LSP: - # For example, avoid cases where we use a GRVW=4 with many empty addresses - # when a GRVW=1 will do instead. - validElementsLoadedPerInst = state["LSC%s"%tc] * state["LSP%s"%tc] - grvw //= 2 - while grvw >= minGrvw: - elementsLoadedPerInst = state["NumThreads"]*grvw//mik - if elementsLoadedPerInst < validElementsLoadedPerInst: - break # Went too far, not enough load elements at this VW - if state["LSC%s"%tc] % grvw == 0: - if dbFract: - print(" stepdown success (valid)elementsLoadedPerInst=", validElementsLoadedPerInst, "/", elementsLoadedPerInst, "grvw=", grvw, "lsc=", state["LSC%s"%tc]) - bestVw = grvw - grvw //= 2 - break - - # TODO - could have this generate dwordx3 loads in addition, step down by 1 instead of div2 - # Would need to change asm code gen to generate x3 - grvw //= 2 - # end-- while loop - - if bestVw == -1: - if dbFract: - print ("reject fractional - no acceptable tile dim? GlobalReadVectorWidth%s"%tc, \ - state["GlobalReadVectorWidth%s"%tc]) - return False # could not find a solution, perhaps only possible for half ? - - state["GlobalReadVectorWidth%s"%tc] = bestVw - if bestVw != state["GlobalReadVectorWidth%s"%tc]: - if dbFract: - print(" reducing GlobalReadVectorWidth%s from %u to %u" \ - % (tc, state["GlobalReadVectorWidth%s"%tc], bestVw)) - - # How many loads per threads in each dimension. - # threads which are outside the global read tile bounds will be clipped - # in the assembly code generator. - # Multiply the LSC*GRVW - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim, state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - nlc = state["NumLoadsCoalesced%s"%tc] - nlp = state["NumLoadsPerpendicular%s"%tc] - - # LoadTile must at least cover the MacroTile: - assert(nlc*state["LSC%s"%tc] >= parDim) - assert(nlp*state["LSP%s"%tc] >= perpDim) - - perpOverhang = perpDim % state["LSP%s"%tc] - state["fractionalPerpOverhang%s"%tc] = perpOverhang - if dbFract: - # how many threads compute Global Read Offsets (GRO) that are not used - print(" PerLoadTile=%ux%u elements Loads/WI=%ux%u LoadTile/WI=%ux%u (MT=%ux%u), %u/%u = %.1f%% WI GRO used %s" \ - % (state["LSC%s"%tc], state["LSP%s"%tc], \ - nlc, nlp, \ - nlc*state["LSC%s"%tc], nlp*state["LSP%s"%tc], \ - parDim, perpDim, \ - parDim*perpDim, \ - nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc], \ - float(parDim*perpDim), \ - float(nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc]) * 100.0) \ - ) - - for p in range(0,nlp): - elementWidth = 4 - if p != nlp-1: - perp = state["LSP%s"%tc] - else: - perp = perpOverhang if perpOverhang else state["LSP%s"%tc] - - validElements = state["LSC%s"%tc] * perp - print(" buffer_load_element_x%u %ux%ux%u bytes, %u/%u valid GRO" %\ - (state["GlobalReadVectorWidth%s"%tc], \ - state["LSC%s"%tc], perp, \ - elementWidth, \ - validElements//state["GlobalReadVectorWidth%s"%tc], - state["NumThreads"])) - - return True - + @staticmethod - def MatrixInstructionToMIParameters(state): + def MatrixInstructionToMIParameters(state, isaInfoMap: Dict[str, IsaInfo]): isa = tuple(state["ISA"]) if len(state["MatrixInstruction"]) == 9: mi = state["MatrixInstruction"] @@ -1737,7 +1555,7 @@ def MatrixInstructionToMIParameters(state): state["MFMA_BF16_1K"] = False if not state["ProblemType"]["Sparse"]: miDataType = state["ProblemType"]["DataType"] if (not state["EnableF32XdlMathOp"]) else state["ProblemType"]["F32XdlMathOp"] - if globalParameters["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not (miDataType.toChar() in validMFMA and \ state["MatrixInstruction"] in validMFMA[miDataType.toChar()]): if miDataType.isBFloat16() and \ @@ -1745,7 +1563,7 @@ def MatrixInstructionToMIParameters(state): state["MFMA_BF16_1K"] = True else: reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], miDataType)) - elif globalParameters["AsmCaps"][isa]["HasWMMA"]: + elif isaInfoMap[isa].asmCaps["HasWMMA"]: if state["MatrixInstruction"] not in validWMMA: reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) else: @@ -1781,7 +1599,7 @@ def MatrixInstructionToMIParameters(state): # set MIInputPerThread isa = tuple(state["ISA"]) state['MIInputPerThread'] = state["MatrixInstruction"][0] * state["MatrixInstruction"][2] * state["MatrixInstruction"][3] // state["WavefrontSize"] - if (not globalParameters["AsmCaps"][isa]['HasMFMA']) and globalParameters["AsmCaps"][isa]['HasWMMA']: + if (not isaInfoMap[isa].asmCaps['HasMFMA']) and isaInfoMap[isa].asmCaps['HasWMMA']: if state['ISA'][0] == 10 or state['ISA'][0] == 11: state['MIInputPerThread'] = state["MatrixInstruction"][2] sparseA = False if not state["ProblemType"]["Sparse"] else False if state["ProblemType"]["Sparse"] == 2 else True @@ -1811,14 +1629,14 @@ def checkAndAssignWaveSeparateGlobalRead(state, tc): ######################################## # determine can we use VgprForLocalReadPacking @staticmethod - def isVgprForLocalReadPackingDoable(state): + def isVgprForLocalReadPackingDoable(state, isaInfoMap: Dict[str, IsaInfo]): isa = tuple(state["ISA"]) doable = True # MatrixInstruction only if not state["EnableMatrixInstruction"]: doable = False # only for HasEccHalf - if not globalParameters["ArchCaps"][isa]["HasEccHalf"]: + if not isaInfoMap[isa].archCaps["HasEccHalf"]: doable = False # only for PLR>=1 (except for DTVA+B) if state["PrefetchLocalRead"] < 1 and not (state["DirectToVgprA"] and state["DirectToVgprB"]): @@ -1839,7 +1657,7 @@ def isDirectToVgprSupportDataType(state): ######################################## # determine can we use DirectToVgpr @staticmethod - def isDirectToVgprDoable(state, tc): + def isDirectToVgprDoable(state, tc, isaInfoMap: Dict[str, IsaInfo]): MIindex = 0 if tc == 'A' else 1 numBytes = state["ProblemType"]["DataType"].numBytes() numBytesGR = state["ProblemType"]["DataType%s"%tc].numBytes() @@ -1896,7 +1714,7 @@ def isDirectToVgprDoable(state, tc): # numBytes < 4 case if state["ProblemType"]["TLU%c"%tc]: # use pack logic (with v_perm) same as local read (only if VgprForLocalReadPacking is doable) - if not Solution.isVgprForLocalReadPackingDoable(state): + if not Solution.isVgprForLocalReadPackingDoable(state, isaInfoMap): reject(state, "Does not meet the requirement for DirectToVgpr%c + TLU%c + numByte < 4"%(tc, tc)) return False # force ClusterLocalRead=1 for DTV + pack @@ -2130,7 +1948,7 @@ def getDivisorName(state, tC): ######################################## # assign all derived parameters @staticmethod - def assignDerivedParameters(state, index, depthuList): + def assignDerivedParameters(state, index, depthuList, isaInfoMap: Dict[str, IsaInfo]): state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -2138,7 +1956,7 @@ def assignDerivedParameters(state, index, depthuList): and (state["ProblemType"]["DataType"].isSingle()): state["EnableF32XdlMathOp"] = True - Solution.assignProblemIndependentDerivedParameters(state) + Solution.assignProblemIndependentDerivedParameters(state, isaInfoMap) if "AssignedDerivedParameters" in state: if state["AssignedDerivedParameters"]: @@ -2189,7 +2007,7 @@ def assignDerivedParameters(state, index, depthuList): reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") if not state["EnableMatrixInstruction"]: reject(state, "Stream-K requires MatrixInstruction") - if globalParameters["AsmCaps"][isa]["HasWMMA"]: + if isaInfoMap[isa].asmCaps["HasWMMA"]: reject(state, "Stream-K untested with WMMA") # if state["PersistentKernel"]: # reject(state, "Cannot enable both Stream-K and PersistentKernel") @@ -2234,13 +2052,13 @@ def assignDerivedParameters(state, index, depthuList): print2("in assignDerivedParameters, state['Valid'] = False") return - if not globalParameters["AsmCaps"][isa]["HasNTModifier"]: + if not isaInfoMap[isa].asmCaps["HasNTModifier"]: # force to disable nt flag if it is not supported by arch for ch in ["", "A", "B", "C", "D", "E", "WS", "Metadata"]: if state["NonTemporal%s"%ch] >= 4: state["NonTemporal%s"%ch] -= 4 - if state["WavefrontSize"] == 32 and not globalParameters["ArchCaps"][isa]["HasWave32"]: + if state["WavefrontSize"] == 32 and not isaInfoMap[isa].archCaps["HasWave32"]: reject(state, "WavefrontSize=32 not supported for ISA {}".format(isa)) return @@ -2249,7 +2067,7 @@ def assignDerivedParameters(state, index, depthuList): return if state["EnableMatrixInstruction"]: - if not (globalParameters["AsmCaps"][isa]["HasMFMA"] or globalParameters["AsmCaps"][isa]["HasWMMA"]): + if not (isaInfoMap[isa].asmCaps["HasMFMA"] or isaInfoMap[isa].asmCaps["HasWMMA"]): reject(state, f"isa {isa} doesn't support matrix instruction") return if not (state["ProblemType"]["DataType"].isSingle() \ @@ -2261,7 +2079,7 @@ def assignDerivedParameters(state, index, depthuList): or state["ProblemType"]["DataType"].isInt8()): reject(state, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) return - if (not globalParameters["AsmCaps"][isa]["HasMFMA"] and globalParameters["AsmCaps"][isa]["HasWMMA"] and (state["WavefrontSize"] == 64)): + if (not isaInfoMap[isa].asmCaps["HasMFMA"] and isaInfoMap[isa].asmCaps["HasWMMA"] and (state["WavefrontSize"] == 64)): print2("!! Warning: WMMA only well tune on WGP mode, wave size = 32") # reject(state, "WMMA only suppport on WGP mode, wave size = 32") # return @@ -2274,14 +2092,14 @@ def assignDerivedParameters(state, index, depthuList): if not state["MIWaveTile"] or len(state["MIWaveTile"]) != 2: reject(state, "invalid MIWaveTile") return - if globalParameters["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not state["ProblemType"]["HighPrecisionAccumulate"] \ and state["ProblemType"]["DataType"].numRegisters() < 1 : reject(state, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ " in fp32 (or i32) precision. Please add the following config:" + \ "\n - HighPrecisionAccumulate: True") return - if globalParameters["AsmCaps"][isa]["HasWMMA"]: + if isaInfoMap[isa].asmCaps["HasWMMA"]: if state["ProblemType"]["DataType"].numRegisters() >=1: reject(state, "WMMA only support half, bf16 and i8 type") return @@ -2534,7 +2352,7 @@ def assignDerivedParameters(state, index, depthuList): if state["KernelLanguage"] == "Assembly" \ and state["ProblemType"]["DataType"].isHalf(): - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if isaInfoMap[globalParameters["CurrentISA"]].archCaps["HasEccHalf"]: if not state["ProblemType"]["HighPrecisionAccumulate"] and state["AssertFree0ElementMultiple"] % 2 != 0: # beta-on-edge has AF0EM requirement except for HPA kernels reject(state, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") @@ -2560,7 +2378,7 @@ def calcLdsPad(lrvw: int) -> int: else: optPadA //= 2 readRegsA //= 2 - if (not globalParameters["AsmCaps"][isa]['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): + if (not isaInfoMap[isa].asmCaps['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): reject(state, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") return if state["EnableMatrixInstruction"]: @@ -3016,7 +2834,7 @@ def calSwizzleK(state, tc): validDepthU = False if validDepthU and state["KernelLanguage"] == "Assembly": - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if isaInfoMap[globalParameters["CurrentISA"]].archCaps["HasEccHalf"]: if state["ProblemType"]["DataType"].numRegisters() == 0.5 and (not state["ProblemType"]["HighPrecisionAccumulate"]): if state["GlobalReadVectorWidthA"] == 1 or state["GlobalReadVectorWidthB"] == 1: reject(state, "HalfEcc requires HPA if glvw = 1") @@ -3212,7 +3030,7 @@ def calSwizzleK(state, tc): if state["AssertFree0ElementMultiple"] < 2: reject(state, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") - if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]['HasWMMA']: + if state["EnableMatrixInstruction"] and isaInfoMap[isa].asmCaps['HasWMMA']: reject(state, "Half WMMA doesn't support single buffer GSU") return @@ -3318,10 +3136,10 @@ def calSwizzleK(state, tc): # Determine if we can load directly-to-Vgpr # need to check after state["LocalReadVectorWidth"] = -1 is resolved if state["DirectToVgprA"]: - if not Solution.isDirectToVgprDoable(state, 'A'): + if not Solution.isDirectToVgprDoable(state, 'A', isaInfoMap): return # rejected if state["DirectToVgprB"]: - if not Solution.isDirectToVgprDoable(state, 'B'): + if not Solution.isDirectToVgprDoable(state, 'B', isaInfoMap): return # rejected ######################################## @@ -3729,13 +3547,13 @@ def subCheckLdsBlockSizePerPad(tc, idx): # check if need to use lds init Acc vgprs state["LdsInitCVgprs"] = False - if globalParameters["ArchCaps"][isa]["HasAccCD"] and \ + if isaInfoMap[isa].archCaps["HasAccCD"] and \ state["EnableMatrixInstruction"] and state["StorePriorityOpt"] and \ state["ProblemType"]["DataType"].isDouble(): state["LdsInitCVgprs"] = True # force MIArchVgpr when using WMMA - if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]["HasWMMA"]: + if state["EnableMatrixInstruction"] and isaInfoMap[isa].asmCaps["HasWMMA"]: state["MIArchVgpr"] = True if state["MIArchVgpr"]: @@ -3743,7 +3561,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): reject(state, "MIArchVgpr only support for MatrixInstruction") return - if globalParameters["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not (state["ProblemType"]["ComputeDataType"].isDouble() or \ state["ProblemType"]["ComputeDataType"].isSingle() or \ (state["ProblemType"]["ComputeDataType"].isHalf() and state["ProblemType"]["HighPrecisionAccumulate"]) or \ diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 81afc23b3b..77d49ae7c0 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -30,7 +30,7 @@ import os import sys import argparse -from .Common import globalParameters, print1, printExit, printWarning, ensurePath, \ +from .Common import globalParameters, print1, printExit, printWarning, ensurePath, IsaInfo, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain @@ -42,6 +42,7 @@ from . import LibraryLogic from datetime import datetime from pathlib import Path +from typing import Dict import subprocess @@ -59,6 +60,7 @@ def executeStepsInConfig( outputPath: Path, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, + isaInfoMap: Dict[str, IsaInfo], cCompiler: str ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -84,7 +86,16 @@ def executeStepsInConfig( # Benchmark Problems ############################################################################## if "BenchmarkProblems" in config: - BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath) + BenchmarkProblems.main( + config["BenchmarkProblems"], + config["UseCache"], + asmToolchain, + srcToolchain, + isaInfoMap, + cCompiler, + outputPath, + buildTmpPath + ) print1("") ############################################################################## @@ -101,7 +112,7 @@ def executeStepsInConfig( libraryLogicConfig = config["LibraryLogic"] else: libraryLogicConfig = {} - LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath) + LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, isaInfoMap, outputPath) print1("") else: print1("# LibraryLogic already done.") @@ -410,7 +421,7 @@ def Tensile(userArgs): store_max_frequency(max_frequency) cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) - assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) + isaInfoMap = assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) @@ -425,7 +436,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 54f93810f2..e6ec8c9730 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -29,13 +29,14 @@ import shutil from pathlib import Path from timeit import default_timer as timer -from typing import List, NamedTuple, Optional, Sequence, Union +from typing import List, NamedTuple, Optional, Sequence, Union, Dict from Tensile import SOURCE_PATH, LibraryIO from Tensile.Common import ( HR, CHeader, IsaVersion, + IsaInfo, ParallelMap2, SemanticVersion, architectureMap, @@ -197,6 +198,7 @@ def writeSolutionsAndKernels( kernels, kernelHelperObjs, kernelWriterAssembly, + isaInfoMap: Dict[str, IsaInfo], errorTolerant=False, generateSourcesAndExit=False, compress=True, @@ -261,7 +263,7 @@ def assemble(ret): asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress ) buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, isaInfoMap, fromTensile ) return codeObjectFiles, numKernels @@ -274,10 +276,10 @@ def writeSolutionsAndKernelsTCL( kernels, kernelHelperObjs, kernelWriterAssembly, + isaInfoMap: Dict[str, IsaInfo], compress=True, fromTensile=False, ): - outputPath = Path(outputPath) destLibPath = ensurePath( outputPath / "library" @@ -326,7 +328,7 @@ def assemble(ret): writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, isaInfoMap, fromTensile ) return len(uniqueAsmKernels) @@ -334,13 +336,13 @@ def assemble(ret): @timing def getSolutionAndKernelWriters( - solutions, kernels, assembler: str, assemblerVersion: SemanticVersion + solutions, kernels, assembler: str, assemblerVersion: SemanticVersion, isaInfoMap: Dict[str, IsaInfo] ): kernelSerialNaming = Solution.getSerialNaming(kernels) solutionMinNaming = Solution.getMinNaming(solutions) kernelMinNaming = Solution.getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion + kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion, isaInfoMap ) return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) @@ -394,7 +396,7 @@ def generateKernelObjectsFromSolutions(solutions): @timing -def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): +def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler, isaInfoMap): if ";" in args["Architecture"]: archs = args["Architecture"].split(";") # user arg list format @@ -405,7 +407,7 @@ def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): masterLibraries = {} nextSolIndex = 0 - fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(archs)) + fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(isaInfoMap), itertools.repeat(archs)) def libraryIter(lib: MasterSolutionLibrary): if len(lib.solutions): @@ -510,7 +512,7 @@ def run(): print1(f"# Architecture(s): {arguments['Architecture']}") print1(f"# Library Format: {arguments['LibraryFormat']}") - assignGlobalParameters(arguments, cxxCompiler) + isaInfoMap = assignGlobalParameters(arguments, cxxCompiler) asmToolchain = AssemblyToolchain( assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"] @@ -573,10 +575,10 @@ def validLogicFile(p: Path): for logicFile in logicFiles: print2("# %s" % logicFile) - solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) + solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler, isaInfoMap) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion + solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion, isaInfoMap ) copyStaticFiles(outputPath) @@ -588,13 +590,14 @@ def validLogicFile(p: Path): kernels, kernelHelperObjs, kernelWriterAssembly, + isaInfoMap, compress=arguments["UseCompression"], ) archs = [ isaToGfx(arch) for arch in globalParameters["SupportedISA"] - if globalParameters["AsmCaps"][arch]["SupportedISA"] + if isaInfoMap[arch].asmCaps["SupportedISA"] ] newLibraryDir = ensurePath(os.path.join(outputPath, "library")) diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index 21511588fe..e11389ea85 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from typing import Tuple -from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs, IsaInfo from .Formatting import __TI_DEBUG_LEVEL__, printExit @@ -48,13 +48,6 @@ def __new__(cls, *args, **kwargs): cls._instance._kernelInfo = {} return cls._instance - @dataclass - class IsaInfo: - assemblerPath: str - asmCaps: dict - archCaps: dict - regCaps: dict - asmBugs: dict @dataclass class kernelInfo: @@ -71,8 +64,7 @@ def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool archCaps = initArchCaps(isaVersion) regCaps = initRegisterCaps(isaVersion, archCaps) asmBugs = initAsmBugs(asmCaps) - self._isaInfo[isaVersion] = TensileInstructions.IsaInfo(assemblerPath, # type: ignore - asmCaps, archCaps, regCaps, asmBugs) + self._isaInfo[isaVersion] = IsaInfo(assemblerPath, asmCaps, archCaps, regCaps, asmBugs) def setDebugLevel(self, level: int) -> None: __TI_DEBUG_LEVEL__ = level diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index 7d1ca2c926..a29b691ec4 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -1,7 +1,10 @@ import math +from typing import Dict from pathlib import Path from inspect import currentframe, getframeinfo +from Tensile.Common import IsaInfo + MI_KEY: str = "MatrixInstruction" MI_ENABLED_KEY: str = "EnableMatrixInstruction" @@ -112,7 +115,7 @@ def elineno(): return f"{Path(frame.filename).name}:{frame.lineno}" -def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): +def validateMatrixInstruction(solution: dict, filepath: Path, isaInfoMap: Dict[str, IsaInfo]) -> bool: """ Validates the matrix instruction configured in the given solution. @@ -137,7 +140,7 @@ def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): AssertionError: If any of the validation checks fail. """ try: - _validateMatrixInstruction(solution, params) + _validateMatrixInstruction(solution, isaInfoMap) return True except AssertionError as e: print(f"Validation failed: {filepath} (index {solution['SolutionIndex']})") @@ -145,7 +148,7 @@ def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): return False -def _validateMatrixInstruction(solution: dict, params: dict): +def _validateMatrixInstruction(solution: dict, isaInfoMap: Dict[str, IsaInfo]): """ Function to validate the matrix instruction for the provided solution. See exported function for more details. @@ -186,15 +189,15 @@ def _validateMatrixInstruction(solution: dict, params: dict): # Check datatype if not isSparse: - if params["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not (miDataType.toChar() in validMFMA and mi in validMFMA[miDataType.toChar()]): assert miDataType.isBFloat16() and mi in validMFMA["B1k"], elineno() - elif params["AsmCaps"][isa]["HasWMMA"]: + elif isaInfoMap[isa].asmCaps["HasWMMA"]: assert mi in validWMMA, elineno() else: assert miDataType.toChar() in validSMFMA and mi in validSMFMA[miDataType.toChar()], elineno() - if (not params["AsmCaps"][isa]["HasMFMA"]) and params["AsmCaps"][isa]["HasWMMA"]: + if (not isaInfoMap[isa].asmCaps["HasMFMA"]) and isaInfoMap[isa].asmCaps["HasWMMA"]: if isa[0] == 10 or isa[0] == 11: assert miInputPerThread == mi[2], elineno() diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 59ee225071..52bab11bfa 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -30,9 +30,9 @@ from pathlib import Path from timeit import default_timer as timer -from typing import List, Union +from typing import List, Union, Dict -from ..Common import globalParameters, print1, print2, ensurePath, splitArchs +from ..Common import globalParameters, print1, print2, ensurePath, splitArchs, IsaInfo class SourceToolchain: def __init__(self, compiler: str, bundler: str, buildIdKind: str, asanBuild: bool=False, saveTemps: bool=False): @@ -170,6 +170,7 @@ def buildSourceCodeObjectFiles( tmpObjDir: Union[Path, str], includeDir: Union[Path, str], kernelPath: Union[Path, str], + isaInfoMap: Dict[str, IsaInfo], fromTensile: bool ) -> List[str]: """Compiles a HIP source code file into a code object file. @@ -197,7 +198,7 @@ def buildSourceCodeObjectFiles( coPathsRaw = [] coPaths= [] - _, cmdlineArchs = splitArchs(globalParameters, fromTensile) + _, cmdlineArchs = splitArchs(globalParameters, isaInfoMap, fromTensile) objPath = str(tmpObjDir / objFilename) toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) From fc23660e9d751a197414a1129cc999d7dc06b514 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:44:48 -0600 Subject: [PATCH 046/105] fix: update capabilities table --- tensilelite/Tensile/Common/Capabilities.py | 2 +- .../Tensile/Common/GlobalParameters.py | 62 ++++++++++--------- tensilelite/Tensile/Components/StreamK.py | 2 +- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index db20079d5a..bfc2b0eb9c 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -164,7 +164,7 @@ def initArchCaps(isaVersion) -> dict: rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) rv["DSLow16NotPreserve"] = isaVersion[0] == (12) - rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) + rv["WorkGroupIdFromTTM"] = isaVersion[0] == (12) rv["NoSDWA"] = isaVersion[0] == (12) rv["VOP3ByteSel"] = isaVersion[0] == (12) rv["HasFP8_OCP"] = isaVersion[0] == (12) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 0f95b6249a..3e767c30b2 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -22,6 +22,7 @@ # ################################################################################ +import itertools import math import os.path import subprocess @@ -29,12 +30,13 @@ import time from collections import OrderedDict from copy import deepcopy +from typing import Dict, List from Tensile import __version__ from .Architectures import gfxToIsa, isaToGfx from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps, initRegisterCaps -from .Types import IsaInfo +from .Types import IsaInfo, IsaVersion from .Utilities import locateExe, versionIsCompatible startTime = time.time() @@ -1572,41 +1574,41 @@ def restoreDefaultGlobalParameters(): globalParameters[key] = value -# def printTable(rows): -# rows = list([[str(cell) for cell in row] for row in rows]) -# colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - -# for row in rows: -# for width, cell in zip(colWidths, row): -# pad = " " * (width - len(cell)) -# print(pad, cell, sep="", end=" ") -# print() +def printCapabilitiesTable(supportedIsas: List[IsaVersion], isaInfoMap: Dict[str, IsaInfo]): + """ + Prints a capability table for the given parameters and ISA information map. + Args: + supportedIsas: The ISAs to show in the table. + isaInfoMap: The ISA information map containing assembler and architecture capabilities. + """ -# def printCapTable(parameters, isaInfoMap): -# import itertools + def printTable(rows): + rows = [[str(cell) for cell in row] for row in rows] + colWidths = [max(len(cell) for cell in col) for col in zip(*rows)] -# archs = [(0, 0, 0)] + parameters["SupportedISA"] -# gfxNames = list(map(isaToGfx, archs)) + for row in rows: + print(" ".join(cell.ljust(width) for cell, width in zip(row, colWidths))) -# headerRow = ["cap"] + gfxNames + def capRow(isaInfoMap, cap, capType): + return [cap] + [ + "1" if cap in getattr(info, capType) and getattr(info, capType)[cap] else "-" + for info in isaInfoMap.values() + ] -# def capRow(caps, cap): -# return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] + gfxs = list(map(isaToGfx, supportedIsas)) + headerRow = ["Capability"] + gfxs -# allAsmCaps = set( -# itertools.chain(*[caps.keys() for arch, caps in isaInfoMap.items()]) -# ) -# allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) -# asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + allAsmCaps = sorted( + set(itertools.chain(*[info.asmCaps for info in isaInfoMap.values()])), + key=lambda k: (k.split("_")[-1], k), + ) + asmCapRows = [capRow(isaInfoMap, cap, "asmCaps") for cap in allAsmCaps] -# allArchCaps = set( -# itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) -# ) -# allArchCaps = sorted(allArchCaps) -# archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + allArchCaps = sorted(set(itertools.chain(*[info.archCaps for info in isaInfoMap.values()]))) + archCapRows = [capRow(isaInfoMap, cap, "archCaps") for cap in allArchCaps] -# printTable([headerRow] + asmCapRows + archCapRows) + printTable([headerRow] + asmCapRows + archCapRows) def assignGlobalParameters(config, cxxCompiler=None): @@ -1712,8 +1714,8 @@ def assignGlobalParameters(config, cxxCompiler=None): asmBugs = initAsmBugs(asmCaps) isaInfoMap[v] = IsaInfo(cxxCompiler, asmCaps, archCaps, regCaps, asmBugs) - # if globalParameters["PrintLevel"] >= 1: - # printCapTable(globalParameters, isaInfoMap) + if globalParameters["PrintLevel"] >= 1: + printCapabilitiesTable(globalParameters["SupportedISA"], isaInfoMap) globalParameters["SupportedISA"] = list( [ diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 73067f64a5..d3a878f1cb 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -1586,7 +1586,7 @@ def preLoop(self, writer, kernel): def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK Off graWorkGroup") - if writer.states.archCaps["WrokGroupIdFromTTM"]: + if writer.states.archCaps["WorkGroupIdFromTTM"]: module.add(SMovB32(dst=sgpr("WorkGroup0"), src="ttmp9", comment="workaround")) module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7")) From 21d99baea0deeb883efef5f861601813c780df54 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 13 Feb 2025 12:52:44 -0600 Subject: [PATCH 047/105] fix: remove asm path from isa info --- tensilelite/Tensile/Common/GlobalParameters.py | 2 +- tensilelite/Tensile/Common/Types.py | 1 - tensilelite/Tensile/TensileInstructions/Base.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 3e767c30b2..8142f49340 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -1712,7 +1712,7 @@ def assignGlobalParameters(config, cxxCompiler=None): archCaps = initArchCaps(v) regCaps = initRegisterCaps(v, archCaps) asmBugs = initAsmBugs(asmCaps) - isaInfoMap[v] = IsaInfo(cxxCompiler, asmCaps, archCaps, regCaps, asmBugs) + isaInfoMap[v] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) if globalParameters["PrintLevel"] >= 1: printCapabilitiesTable(globalParameters["SupportedISA"], isaInfoMap) diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index ad415caeb8..c17be22046 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -5,7 +5,6 @@ @dataclass class IsaInfo: - assemblerPath: str asmCaps: dict archCaps: dict regCaps: dict diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index e11389ea85..d01638c80f 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -64,7 +64,7 @@ def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool archCaps = initArchCaps(isaVersion) regCaps = initRegisterCaps(isaVersion, archCaps) asmBugs = initAsmBugs(asmCaps) - self._isaInfo[isaVersion] = IsaInfo(assemblerPath, asmCaps, archCaps, regCaps, asmBugs) + self._isaInfo[isaVersion] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) def setDebugLevel(self, level: int) -> None: __TI_DEBUG_LEVEL__ = level From 668901780791fb9df883e7def5f90381e415f1ab Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 13 Feb 2025 14:56:51 -0600 Subject: [PATCH 048/105] fix: more missing function arguments --- tensilelite/Tensile/BenchmarkProblems.py | 8 +++++--- tensilelite/Tensile/Tensile.py | 2 +- tensilelite/Tensile/TensileInstructions/Base.py | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 91acf5a29b..4e3ee22375 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -82,12 +82,14 @@ def getCustomKernelSolutionObj( return Solution(config, cxxCompiler, isaInfoMap) -def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str): +def generateCustomKernelSolutions( + problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo] + ): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler) + solution = getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, isaInfoMap) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -293,7 +295,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, isaInfoMap) kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, srcToolchain.compiler) + not benchmarkStep.customKernelWildcard, srcToolchain.compiler, isaInfoMap) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 77d49ae7c0..bf53b6f94a 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -126,7 +126,7 @@ def executeStepsInConfig( libraryClientConfig = config["LibraryClient"] else: libraryClientConfig = {} - ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath) + ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, isaInfoMap, outputPath) print1("") diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index d01638c80f..e2ed43eab9 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from typing import Tuple -from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs, IsaInfo +from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs, IsaInfo, IsaVersion from .Formatting import __TI_DEBUG_LEVEL__, printExit @@ -51,10 +51,10 @@ def __new__(cls, *args, **kwargs): @dataclass class kernelInfo: - isa: Tuple[int, int, int] + isa: IsaVersion wavefrontSize: int = 64 - def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool=False) -> None: + def init(self, isaVersion: IsaVersion, assemblerPath: str, debug: bool=False) -> None: with self._lock: if len(self._kernelInfo) > 1000: self._kernelInfo = _removeIdent(self._kernelInfo) From f6effac583782fa51ed9054379c384dec99b1912 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Thu, 13 Feb 2025 23:42:25 +0000 Subject: [PATCH 049/105] Remove currentISA from globalParameters --- tensilelite/Tensile/BenchmarkProblems.py | 61 ++- tensilelite/Tensile/ClientWriter.py | 11 +- tensilelite/Tensile/Common/Architectures.py | 81 ++- tensilelite/Tensile/Common/Capabilities.py | 4 +- .../Tensile/Common/GlobalParameters.py | 99 +--- tensilelite/Tensile/Common/Types.py | 6 +- tensilelite/Tensile/Common/Utilities.py | 61 +-- tensilelite/Tensile/Contractions.py | 12 +- tensilelite/Tensile/KernelWriter.py | 12 +- .../KernelWriterActivationEnumHeader.py | 1 - .../Tensile/KernelWriterActivationFunction.py | 20 +- tensilelite/Tensile/KernelWriterAssembly.py | 7 +- tensilelite/Tensile/KernelWriterBetaOnly.py | 2 +- tensilelite/Tensile/KernelWriterConversion.py | 15 +- tensilelite/Tensile/LibraryIO.py | 26 +- tensilelite/Tensile/LibraryLogic.py | 8 +- tensilelite/Tensile/Ops/AMaxGenerator.py | 3 +- tensilelite/Tensile/Ops/LayerNormGenerator.py | 3 +- tensilelite/Tensile/Ops/SoftmaxGenerator.py | 3 +- tensilelite/Tensile/SolutionLibrary.py | 15 +- .../Tensile/SolutionSelectionLibrary.py | 5 +- tensilelite/Tensile/SolutionStructs.py | 495 +++++++++--------- tensilelite/Tensile/Tensile.py | 23 +- .../Tensile/TensileCreateLibrary/Run.py | 43 +- .../Tensile/TensileInstructions/Base.py | 14 +- tensilelite/Tensile/Toolchain/Source.py | 6 +- 26 files changed, 505 insertions(+), 531 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 26f49968aa..847fcfb7de 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -30,6 +30,7 @@ from copy import deepcopy from pathlib import Path +from typing import List from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from Tensile.BenchmarkStructs import BenchmarkProcess, constructForkPermutations @@ -44,10 +45,11 @@ from Tensile.Toolchain.Source import SourceToolchain from Tensile.Common import globalParameters, HR, print1, print2, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ - BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx -def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, debugConfig: DebugConfig): +def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, \ + debugConfig: DebugConfig, currentIsa: IsaVersion): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -59,7 +61,7 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo solution.update(perm) # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, debugConfig.splitGSU, cxxCompiler) + solutionObject = Solution(solution, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, [currentIsa], cxxCompiler) if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) @@ -70,18 +72,20 @@ def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCo return solutions -def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, directory=CUSTOM_KERNEL_PATH): +def _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, debugConfig: DebugConfig, \ + currentIsa: IsaVersion, directory=CUSTOM_KERNEL_PATH): """Creates the Solution object for a custom kernel""" config = getCustomKernelConfig(kernelName, internalSupportParams, directory) - return Solution(config, cxxCompiler) + return Solution(config, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, [currentIsa], cxxCompiler) -def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str, printSolutionRejectionReason: bool): +def _generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, \ + cxxCompiler: str, debugConfig: DebugConfig, currentIsa: IsaVersion): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, currentIsa, debugConfig) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -107,7 +111,7 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar print1("# Added {} to solutions".format(kernelName)) if solution["Valid"]: solutions.append(solution) - elif printSolutionRejectionReason: + elif debugConfig.printSolutionRejectionReason: print1("rejecting solution " + str(solution)) return solutions @@ -115,8 +119,7 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, - useShortNames: bool, - debugConfig: DebugConfig): + useShortNames: bool, debugConfig: DebugConfig, currentIsa: IsaVersion, deviceId: int): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -145,21 +148,22 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, debugConfig) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, debugConfig, currentIsa) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ sourcePath, asmToolchain, srcToolchain, \ solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, debugConfig.splitGSU, errorTolerant=True, fromTensile=True, \ + kernelWriterAssembly, debugConfig.splitGSU, \ + [isaToGfx(currentIsa)], errorTolerant=True, \ generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], \ useShortNames=useShortNames) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") - newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler, debugConfig.splitGSU) + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, [currentIsa]) newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) @@ -190,11 +194,11 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ idealProblemSizes = ProblemSizes(problemType, idealSizes) writeClientConfig(True, solutions, idealProblemSizes, biasTypeArgs, \ factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, True) + newLibrary, codeObjectFiles, True, deviceId) else: writeClientConfig(True, solutions, problemSizes, biasTypeArgs, \ factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, False) + newLibrary, codeObjectFiles, False, deviceId) if len(solutions) == 0: printExit("write solutions and kernels results 0 valid soultion.") @@ -202,9 +206,10 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ return codeObjectFiles -def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, +def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, debugConfig: DebugConfig + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, + debugConfig: DebugConfig, currentIsa: IsaVersion, deviceId: int ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -284,11 +289,13 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.paramGroups) if problemSizeGroupConfig["ForkParameters"] else [] maxPossibleSolutions = len(forkPermutations) - regSolutions = generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, debugConfig) - kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ + regSolutions = _generateForkedSolutions(benchmarkProcess.problemType, \ + benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, \ + debugConfig, currentIsa) + kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, srcToolchain.compiler, debugConfig.printSolutionRejectionReason) + not benchmarkStep.customKernelWildcard, srcToolchain.compiler, debugConfig, \ + currentIsa) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions @@ -318,7 +325,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath, useShortNames, debugConfig) + sourcePath, useShortNames, debugConfig, currentIsa, deviceId) # ^ this mutates solutions # write cache data @@ -354,7 +361,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, benchmarkStep.icacheFlushArgs, conProblemType, stepBaseDir, codeObjectFiles, resultsFileName, - outFile) + outFile, deviceId) # I think the size portion of this yaml could be removed, # but for now it's needed, so we update it even in the cache case @@ -384,7 +391,8 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, \ - cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool, debugConfig: DebugConfig): + cCompiler: str, outputPath: Path, buildTmpPath: Path, useShortNames: bool, \ + debugConfig: DebugConfig, currentIsa: IsaVersion, deviceId: int): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) @@ -423,9 +431,10 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # benchmark problem size group benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, \ + _benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, \ asmToolchain, srcToolchain, cCompiler, buildTmpPath, \ - benchmarkProblemsPath, useShortNames, debugConfig) + benchmarkProblemsPath, useShortNames, debugConfig, \ + currentIsa, deviceId) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index aa5aac2f1b..6c50b4260e 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -500,7 +500,7 @@ def pruneModeName(mode): if mode == 5: return 'Prune0X0X' if mode == 6: return 'Prune00XX' -def writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, libraryFile=None): +def writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, deviceId: int, libraryFile=None): assert os.path.exists(sourceDir), f"sourceDir={sourceDir} does not exist" @@ -573,7 +573,7 @@ def param(key, value): if globalParameters["DataInitValueActivationArgs"]: param('activation-additional-args', ','.join(map(str, globalParameters["DataInitValueActivationArgs"]))) - param("device-idx", globalParameters["Device"]) + param("device-idx", deviceId) param("init-seed", globalParameters["DataInitSeed"]) @@ -656,6 +656,7 @@ def writeClientConfig( newLibrary, codeObjectFiles, tileAwareSelection, + deviceId: int, configBase = "ClientParameters", libraryFile = None ): @@ -677,11 +678,11 @@ def writeClientConfig( resultsFileName = os.path.join(stepBaseDir, "../Data", stepName+".csv") newSolution = next(iter(newLibrary.solutions.values())) - writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, libraryFile) + writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, deviceId, libraryFile) return filename -def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataFilePath, configFile, problemTypeDict=None): +def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataFilePath, configFile, deviceId, problemTypeDict=None): libraryPath = os.path.join(libraryRootPath, "library") libraryFiles = [os.path.join(libraryPath, f) for f in os.listdir(libraryPath)] @@ -698,4 +699,4 @@ def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataF problemTypeDict = metaData["ProblemType"] problemType = ContractionsProblemType.FromOriginalState(problemTypeDict) - writeClientConfigIni(True, problemSizes, "", "", "", "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile) + writeClientConfigIni(True, problemSizes, "", "", "", "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile, deviceId) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index 13909cce4e..eebbc7a7c1 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -21,9 +21,11 @@ ################################################################################ import re -from typing import Optional +from subprocess import run, PIPE +from typing import List, Optional, Tuple from .Types import IsaVersion +from .Utilities import locateExe # Translate GPU targets to filter filenames in Tensile_LOGIC directory architectureMap = { @@ -61,6 +63,27 @@ } +SUPPORTED_ISA = [ + IsaVersion(8, 0, 3), + IsaVersion(9, 0, 0), + IsaVersion(9, 0, 6), + IsaVersion(9, 0, 8), + IsaVersion(9, 0, 10), + IsaVersion(9, 4, 0), + IsaVersion(9, 4, 1), + IsaVersion(9, 4, 2), + IsaVersion(10, 1, 0), + IsaVersion(10, 1, 1), + IsaVersion(10, 1, 2), + IsaVersion(10, 3, 0), + IsaVersion(11, 0, 0), + IsaVersion(11, 0, 1), + IsaVersion(11, 0, 2), + IsaVersion(12, 0, 0), + IsaVersion(12, 0, 1), +] + + def gfxToIsa(name: str) -> Optional[IsaVersion]: """Extracts the ISA version from a given gfx architecture name. @@ -82,7 +105,7 @@ def gfxToIsa(name: str) -> Optional[IsaVersion]: ipart = ipart[:-1] major = int(ipart) - return tuple((major, minor, step)) + return IsaVersion(major, minor, step) def isaToGfx(arch: IsaVersion) -> str: @@ -116,3 +139,57 @@ def gfxToSwCodename(gfxName: str) -> Optional[str]: if gfxName in archKey: return architectureMap[archKey] return None + + +def cliArchsToIsa(cliArchs: str) -> List[IsaVersion]: + """Maps the requested gfx architectures to ISA numbers. + + Args: + archs: str of ";" or "_" separated gfx architectures (e.g., gfx1100 or gfx90a;gfx1101). + + Returns: + List of tuples + """ + archs = cliArchs.split(";") if ";" in cliArchs else cliArchs.split("_") + return SUPPORTED_ISA if "all" in archs else [gfxToIsa(''.join(map(str, arch))) for arch in archs] + + +def _detectGlobalCurrentISA(detectionTool, deviceId: int): + """ + Returns returncode if detection failure + """ + process = run([detectionTool], stdout=PIPE) + archList = [] + for line in process.stdout.decode().split("\n"): + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in SUPPORTED_ISA: + print("# Detected local GPU with ISA: " + isaToGfx(arch)) + archList.append(arch) + if len(archList) > 0: + result = archList[deviceId] + if process.returncode: + print(f"{detectionTool} exited with code {process.returncode}") + return result if process.returncode == 0 else process.returncode + + +# locateExe silently fails which is not good +AMDGPUArchPath = locateExe( + "/opt/rocm", "llvm/bin/amdgpu-arch" +) + +ROCmAgentEnumeratorPath = locateExe( + "/opt/rocm/bin","rocm_agent_enumerator" +) + +def detectGlobalCurrentISA(deviceId: int): + """ + Returns returncode if detection failure + """ + result = _detectGlobalCurrentISA(AMDGPUArchPath, deviceId) + if not isinstance(result, IsaVersion): + print("Attempting to detect ISA with rocm_agent_enumerator") + result = _detectGlobalCurrentISA(ROCmAgentEnumeratorPath, deviceId) + if not isinstance(result, IsaVersion): + raise Exception("Failed to detect currect ISA") + return result diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index db20079d5a..b372f7d9e8 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -3,10 +3,11 @@ from typing import Tuple from .Architectures import isaToGfx +from .Types import IsaVersion def _tryAssembler( - isaVersion: Tuple[int, int, int], + isaVersion: IsaVersion, assemblerPath: str, asmString: str, debug: bool = False, @@ -31,7 +32,6 @@ def _tryAssembler( *options, "-", ] - result = subprocess.run( args, input=asmString.encode(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index aefe7fef28..f15b48c802 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -32,8 +32,9 @@ from Tensile import __version__ -from .Architectures import gfxToIsa, isaToGfx +from .Architectures import isaToGfx, SUPPORTED_ISA, detectGlobalCurrentISA from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps +from .Types import IsaVersion from .Utilities import locateExe, versionIsCompatible startTime = time.time() @@ -232,32 +233,11 @@ # device selection globalParameters["Platform"] = 0 # select opencl platform -globalParameters["Device"] = 0 # select hip device or opencl device within platform # shouldn't need to change globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use -globalParameters["SupportedISA"] = [ - (8, 0, 3), - (9, 0, 0), - (9, 0, 6), - (9, 0, 8), - (9, 0, 10), - (9, 4, 0), - (9, 4, 1), - (9, 4, 2), - (10, 1, 0), - (10, 1, 1), - (10, 1, 2), - (10, 3, 0), - (11, 0, 0), - (11, 0, 1), - (11, 0, 2), - (12, 0, 0), - (12, 0, 1), -] # assembly kernels writer supports these architectures - globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. globalParameters["ClientExecutionLockPath"] = ( None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. @@ -283,7 +263,7 @@ globalParameters["RuntimeLanguage"] = "HIP" globalParameters["CodeObjectVersion"] = "4" -globalParameters["Architecture"] = "all" +#globalParameters["Architecture"] = "all" # perf model globalParameters["PerfModelL2ReadHits"] = 0.0 @@ -401,7 +381,7 @@ ] validMacroTiles = [] validISA = [(0, 0, 0)] -validISA.extend(globalParameters["SupportedISA"]) +validISA.extend(SUPPORTED_ISA) depthUs = list(range(2, 1024 + 1, 1)) for i in validMacroTileSides: for j in validMacroTileSides: @@ -1490,52 +1470,6 @@ def printExit(message): # a yaml file is compatible with tensile if # tensile.major == yaml.major and tensile.minor.step > yaml.minor.step ################################################################################ -def detectGlobalCurrentISA_(detectionTool): - """ - Returns returncode if detection failure - """ - global globalParameters - - if globalParameters["CurrentISA"] == (0, 0, 0) and detectionTool: - process = subprocess.run([detectionTool], stdout=subprocess.PIPE) - if os.name == "nt": - line = "" - for line_in in process.stdout.decode().splitlines(): - if "gcnArchName" in line_in: - line += line_in.split()[1] - break # detemine if hipinfo will support multiple arch - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - globalParameters["CurrentISA"] = arch - else: - archList = [] - for line in process.stdout.decode().split("\n"): - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - archList.append(arch) - if len(archList) > 0: - globalParameters["CurrentISA"] = archList[globalParameters["Device"]] - if process.returncode: - printWarning("%s exited with code %u" % (detectionTool, process.returncode)) - return process.returncode - return 0 - - -def detectGlobalCurrentISA(): - """ - Returns returncode if detection failure - """ - errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) - if errorCode: - printWarning("Attempting to detect ISA with rocm_agent_enumerator") - return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) - return errorCode - - def restoreDefaultGlobalParameters(): """ Restores `globalParameters` back to defaults. @@ -1563,7 +1497,7 @@ def printTable(rows): def printCapTable(parameters): import itertools - archs = [(0, 0, 0)] + parameters["SupportedISA"] + archs = [(0, 0, 0)] + SUPPORTED_ISA gfxNames = list(map(isaToGfx, archs)) headerRow = ["cap"] + gfxNames @@ -1593,7 +1527,7 @@ def assignGlobalParameters(config, cxxCompiler=None): can override them, those overridings happen here """ - global globalParameters + global globalParameters, SUPPORTED_ISA # Minimum Required Version if "MinimumRequiredVersion" in config: @@ -1670,38 +1604,39 @@ def assignGlobalParameters(config, cxxCompiler=None): globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] # read current gfx version - returncode = detectGlobalCurrentISA() + currentIsa = detectGlobalCurrentISA(0) + globalParameters["CurrentISA"] = currentIsa if globalParameters["CurrentISA"] == (0, 0, 0): printWarning( "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." - % globalParameters["SupportedISA"] + % SUPPORTED_ISA ) - if returncode: - if os.name == "nt": - globalParameters["CurrentISA"] = (9, 0, 6) - printWarning("Failed to detect ISA so forcing (gfx906) on windows") globalParameters["AsmCaps"] = {} globalParameters["ArchCaps"] = {} globalParameters["AsmBugs"] = {} - for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: + # We shouldn't need to do this for all ISAs... + # Why not only do this for ISAs that we are building. + for v in SUPPORTED_ISA + [IsaVersion(0, 0, 0)]: + globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) globalParameters["ArchCaps"][v] = initArchCaps(v) globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + if globalParameters["PrintLevel"] >= 1: printCapTable(globalParameters) - globalParameters["SupportedISA"] = list( + SUPPORTED_ISA = list( [ i - for i in globalParameters["SupportedISA"] + for i in SUPPORTED_ISA if globalParameters["AsmCaps"][i]["SupportedISA"] ] ) - validParameters["ISA"] = [(0, 0, 0), *globalParameters["SupportedISA"]] + validParameters["ISA"] = [(0, 0, 0), *SUPPORTED_ISA] # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index bd040c4200..93bf34c49e 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -1,13 +1,11 @@ from typing import NamedTuple, Tuple -IsaVersion = Tuple[int, int, int] - - class SemanticVersion(NamedTuple): major: int minor: int patch: int - + +IsaVersion = SemanticVersion class DebugConfig(NamedTuple): enableAsserts: bool=False enableDebugA: bool=False diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 291ad52c28..db99b2c87f 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -1,17 +1,12 @@ import functools import math import os -import re import sys import time from enum import Enum -from typing import List, Tuple from Tensile import __version__ -from .Architectures import isaToGfx - - # get param values from structures. def hasParam(name, structure): if isinstance(structure, list): @@ -39,64 +34,10 @@ def locateExe(defaultPath, exeName): # /opt/rocm/bin, hip-clang exePath = os.path.join(path, exeName) if isExe(exePath): return exePath + # if we reach this point we should at least warn and maybe fail return None -def splitArchs(params: dict, fromTensile=False) -> Tuple[List[str], List[str]]: - """ - Splits and processes the architecture strings based on the provided parameters. - - Args: - params: A dictionary of global parameters. - fromTensile: A flag indicating if the function is called from the context of Tensile. - - Returns: - A tuple containing two lists: - - archs: A list of architecture strings with ``-`` instead of ``:`` - - cmdlineArchs: A list of architecture strings that retain ``:`` characters. - """ - - def isSupported(arch): - return ( - params["AsmCaps"][arch]["SupportedISA"] and params["AsmCaps"][arch]["SupportedSource"] - ) - - if ";" in params["Architecture"]: - wantedArchs = params["Architecture"].split(";") - else: - wantedArchs = params["Architecture"].split("_") - archs = [] - cmdlineArchs = [] - if "all" in wantedArchs: - for arch in params["SupportedISA"]: - if isSupported(arch): - if arch in [(9, 0, 6), (9, 0, 8), (9, 0, 10), (9, 4, 0), (9, 4, 1), (9, 4, 2)]: - if arch == (9, 0, 10): - archs += [isaToGfx(arch) + "-xnack+"] - cmdlineArchs += [isaToGfx(arch) + ":xnack+"] - if params["AsanBuild"]: - archs += [isaToGfx(arch) + "-xnack+"] - cmdlineArchs += [isaToGfx(arch) + ":xnack+"] - else: - archs += [isaToGfx(arch) + "-xnack-"] - cmdlineArchs += [isaToGfx(arch) + ":xnack-"] - else: - archs += [isaToGfx(arch)] - cmdlineArchs += [isaToGfx(arch)] - else: - for arch in wantedArchs: - archs += [re.sub(":", "-", arch)] - cmdlineArchs += [arch] - - # if calling from the context of Tensile we only want the arch associated with the current ISA - if fromTensile: - gfx = isaToGfx(params["CurrentISA"]) - archs = set(a for a in archs if gfx in a) - cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) - - return archs, cmdlineArchs - - def ensurePath(path): try: os.makedirs(path) diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 129f378342..83813d3267 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -28,7 +28,8 @@ from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering +from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering, \ + IsaVersion @state_key_ordering class FreeIndex: @@ -656,14 +657,13 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, cxxCompiler: str): - return cls.FromOriginalState(solution._state, cxxCompiler, solution.srcName) + def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: IsaVersion, cxxCompiler: str, ): + return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, solution.srcName) @classmethod - def FromOriginalState(cls, d, cxxCompiler, splitGSU: bool, srcName = "", deviceInfo=None): + def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: IsaVersion, cxxCompiler, srcName = "", deviceInfo=None): rv = cls() - if 'SolutionNameMin' in d: rv.name = d['SolutionNameMin'] @@ -707,7 +707,7 @@ def FromOriginalState(cls, d, cxxCompiler, splitGSU: bool, srcName = "", deviceI d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, splitGSU, cxxCompiler, srcName) + rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, srcName) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 6dd84f9e7f..db23032b2a 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -37,7 +37,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import globalParameters, printWarning, roundUp, print2, DebugConfig, DataDirection, SemanticVersion, \ - INDEX_CHARS, MAX_FILENAME_LENGTH + INDEX_CHARS, MAX_FILENAME_LENGTH, IsaVersion import abc import os @@ -358,13 +358,14 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig, currentIsa: IsaVersion): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler - self.amdClangVersion = amdClangVersion + self.amdClangVersion = amdClangVersion # this is a bug self.ti = None self.debugConfig = debugConfig + self.currentIsa = currentIsa self.do = {} self.do["PreLoop"] = True @@ -4975,7 +4976,8 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): self.states.kernel = kernel self.states.language = "ASM" # we already do this in the solution ctor - self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] + #self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] + self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else self.currentIsa if not globalParameters["AsmCaps"][self.states.version]["SupportedISA"]: self.states.version = (9,0,0) printWarning(f"ISA: {self.version} is not supported; overriding with {self.states.version}") @@ -5016,7 +5018,7 @@ def getKernelFileBase(self, useShortNames: bool, kernel): return fileBase def getKernelName(self, kernel): - kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, True, self.debugConfig.splitGSU) + kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, self.debugConfig.splitGSU, True) return kernelName @abc.abstractmethod diff --git a/tensilelite/Tensile/KernelWriterActivationEnumHeader.py b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py index 84ba70c27b..b7b4f563f0 100644 --- a/tensilelite/Tensile/KernelWriterActivationEnumHeader.py +++ b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py @@ -22,7 +22,6 @@ from copy import deepcopy -from .Common import globalParameters, CHeader from .Activation import ActivationType from .KernelWriterBase import KernelWriterBase diff --git a/tensilelite/Tensile/KernelWriterActivationFunction.py b/tensilelite/Tensile/KernelWriterActivationFunction.py index e25a4ca905..ef70b78bf1 100644 --- a/tensilelite/Tensile/KernelWriterActivationFunction.py +++ b/tensilelite/Tensile/KernelWriterActivationFunction.py @@ -21,15 +21,16 @@ ################################################################################ from copy import deepcopy +from typing import List from .TensileInstructions import TensileInstructions -from .Common import globalParameters, gfxToIsa, isaToGfx +from .Common import isaToGfx, IsaVersion from .Activation import ActivationInline, ActivationType from .KernelWriterBase import KernelWriterBase class KernelWriterActivationFunction(KernelWriterBase): - def __init__(self, state, cxxCompiler: str): + def __init__(self, state, cxxCompiler: str, supportedISA: List[IsaVersion]): super().__init__() self.cxxCompiler = cxxCompiler self.state["ProblemType"] = deepcopy(state["ProblemType"]) @@ -46,16 +47,7 @@ def __init__(self, state, cxxCompiler: str): self.enumName = "Tensile::%sActivationType_%s"%(self.actGradientPrefix, \ self.state["ProblemType"]["ActivationComputeDataType"]) - # Get supported archs - if ";" in globalParameters["Architecture"]: - self.supportedArchs = globalParameters["Architecture"].split(";") - else: - self.supportedArchs = globalParameters["Architecture"].split("_") - if "all" in self.supportedArchs: - self.supportedArchs = deepcopy(globalParameters['SupportedISA']) - else: - for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) + self.supportedArchs = supportedISA # derive parameter self.language = "HIP" @@ -93,11 +85,11 @@ def functionSignature(self): def getInlineAsm(self, activation: ActivationInline, spaces: int, activationType: str): activationStrList = [] - isa = tuple(self.state["Kernel"]["ISA"]) + isa = self.state["Kernel"]["ISA"] if not self._tf.isInit(): self._tf.init(isa, self.cxxCompiler) self._tf.setKernelInfo(isa, self.state["Kernel"]["WavefrontSize"]) - + for arch in self.supportedArchs: self._tf.init(arch, self.cxxCompiler) self._tf.setKernelInfo(arch, self.state["Kernel"]["WavefrontSize"]) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index d1fb73a6be..657b58b58e 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,7 +48,8 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, SemanticVersion +from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, \ + SemanticVersion, IsaVersion from math import ceil, log, floor from copy import deepcopy @@ -67,8 +68,8 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig) + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig, currentIsa: IsaVersion): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig, currentIsa) def getSourceFileString(self, kernel, useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" diff --git a/tensilelite/Tensile/KernelWriterBetaOnly.py b/tensilelite/Tensile/KernelWriterBetaOnly.py index c3f750c361..b49a856a0e 100644 --- a/tensilelite/Tensile/KernelWriterBetaOnly.py +++ b/tensilelite/Tensile/KernelWriterBetaOnly.py @@ -24,7 +24,7 @@ from copy import deepcopy -from .Common import globalParameters, CHeader, INDEX_CHARS +from .Common import INDEX_CHARS from .TensileInstructions import DataType from .KernelWriterBase import KernelWriterBase diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 582205c7cf..d7b48718c6 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -23,15 +23,16 @@ ################################################################################ from copy import deepcopy +from typing import List from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, gfxToIsa, isaToGfx, INDEX_CHARS +from .Common import globalParameters, isaToGfx, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): - def __init__(self, state, load_vw): + def __init__(self, state, load_vw, supportedArchs: List[tuple]): super().__init__() self.state["ProblemType"] = deepcopy(state["ProblemType"]) @@ -80,15 +81,7 @@ def __init__(self, state, load_vw): self.tileChar1 = self.indexChars[self.state["ProblemType"]["Index1"]] # Get supported archs - if ";" in globalParameters["Architecture"]: - self.supportedArchs = globalParameters["Architecture"].split(";") - else: - self.supportedArchs = globalParameters["Architecture"].split("_") - if "all" in self.supportedArchs: - self.supportedArchs = deepcopy(globalParameters['SupportedISA']) - else: - for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) + self.supportedArchs = supportedArchs self.gsuKernels = [self.state["GlobalSplitU"]] if self.state["GenPGRPostKernels"]: diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 22bcf25a7a..fe5395cdfb 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -26,7 +26,8 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__ +from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__, \ + IsaVersion from typing import NamedTuple, List import os @@ -192,12 +193,12 @@ def readJson(filename): data = json.loads(f.read()) return data -def parseSolutionsFile(filename, cxxCompiler, splitGSU: bool): +def parseSolutionsFile(filename, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool): """Wrapper function to read and parse a solutions file.""" - return parseSolutionsData(read(filename), filename, cxxCompiler, splitGSU) + return parseSolutionsData(read(filename), filename, cxxCompiler, splitGSU, printSolutionRejectionReason) -def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool): +def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool): """Parses problem sizes and solutions from the data of a solutions file.""" if len(data) < 3: printExit("Solution file {} is missing required fields (len = {} < 3" \ @@ -224,7 +225,7 @@ def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool): # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, splitGSU, cxxCompiler, srcFile) + solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, [solutionState["ISA"]], cxxCompiler, srcFile) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) @@ -240,15 +241,17 @@ class LibraryLogic(NamedTuple): exactLogic: list library: SolutionLibrary.MasterSolutionLibrary -def parseLibraryLogicFile(filename, cxxCompiler, archs=None): +def parseLibraryLogicFile(filename, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, archs): """Wrapper function to read and parse a library logic file.""" - return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, archs) + return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, splitGSU, printSolutionRejectionReason, archs) -def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): +def parseLibraryLogicData(data, srcFile, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, archs): """Parses the data of a library logic file.""" if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) + + supportedISA = [gfxToIsa(a) for a in archs] is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") if not (archs is None) and "ArchitectureName" in data: @@ -275,7 +278,7 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: if solutionState["KernelLanguage"] == "Assembly": solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) else: - solutionState["ISA"] = (0, 0, 0) + solutionState["ISA"] = IsaVersion(0, 0, 0) solutionState["CUCount"] = data["CUCount"] # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False @@ -290,7 +293,8 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + + solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, srcFile) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -304,7 +308,7 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: solutions = [solutionStateToSolution(solutionState, cxxCompiler) for solutionState in data["Solutions"]] - newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, cxxCompiler) + newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler) return LibraryLogic(data["ScheduleName"], data["ArchitectureName"], problemType, solutions, \ data.get("ExactLogic"), newLibrary) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 1eb30a62cb..a51642a147 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -1430,7 +1430,7 @@ def indicesToSerial(self, solutionIdx, indices ): return serial -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool): +def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool, printSolutionRejectionReason: bool): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1473,7 +1473,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU, printSolutionRejectionReason) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1546,7 +1546,7 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool): +def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool, printSolutionRejectionReason: bool): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU) + generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU, printSolutionRejectionReason) diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index 79682e8c4f..133fd47fe5 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -848,8 +848,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA() arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index 2df20f2d17..4016fa1a71 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -925,8 +925,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA() arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index 34ec8c0cd2..a6ce873b36 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -692,8 +692,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA() arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 835d3badab..76fb78fe87 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -23,13 +23,14 @@ ################################################################################ import itertools +from typing import List from . import Properties from . import Hardware from . import Common from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state +from .Common import state, IsaVersion class SingleSolutionLibrary: Tag = "Single" @@ -300,6 +301,9 @@ def FixSolutionIndices(cls, solutions): def FromOriginalState(cls, origData, origSolutions, + splitGSU: bool, + printSolutionRejectionReason: bool, + supportedISA: List[IsaVersion], cxxCompiler, solutionClass=Contractions.Solution, libraryOrder=None, @@ -460,6 +464,9 @@ def selection(d, problemType, solutions, library, placeholderName): lazyLibrary, placeholderName = \ MasterSolutionLibrary.FromOriginalState(origData, origSolutions, + splitGSU, + printSolutionRejectionReason, + supportedISA, cxxCompiler, solutionClass, libraryOrder[placeholderIndex:], @@ -468,7 +475,7 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, cxxCompiler) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -489,8 +496,8 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler, splitGSU) for s in solutions]) + def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: List[IsaVersion]): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler) for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py index bafe0c0755..6df4ea21f0 100644 --- a/tensilelite/Tensile/SolutionSelectionLibrary.py +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -89,8 +89,9 @@ def updateValidSolutions(validSolutions, analyzerSolutions, solutionMinNaming): (validSolution, validSolutionInfo) = validSelectionSolution selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, True) + splitGSU = False # this is a reminder that we need to add this in to the function signature + validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 0d27a07598..1f07bdc298 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -32,10 +32,10 @@ from .KernelWriterActivationOnly import KernelWriterActivationOnly from .KernelWriterReduction import KernelWriterReduction -from .AsmStoreState import VectorDataTypes from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig +from .AsmStoreState import VectorDataTypes from .Common import assignParameterWithDefault, \ defaultProblemType, defaultSolution, \ @@ -43,13 +43,14 @@ globalParameters, internalParameters, \ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS + validGEMMTypes, HPATypes, roundUp, validWMMA, \ + INDEX_CHARS, IsaVersion, print1 from collections import OrderedDict from collections.abc import Mapping from enum import Enum from functools import lru_cache -from typing import List, NamedTuple +from typing import List import collections import math @@ -61,7 +62,6 @@ def reject(state, printSolutionRejectionReason: bool, *args): if state and "NoReject" in state and state["NoReject"]: return - if printSolutionRejectionReason: sys.stdout.write("\nreject: ") for a in args: @@ -76,7 +76,7 @@ def reject(state, printSolutionRejectionReason: bool, *args): solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None # if we don't have SolutionNameMin, we simply use the problemTypeName solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin - print("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ + raise Exception("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) if state != None: state["Valid"] = False @@ -1052,8 +1052,8 @@ def isExtractableIndex(ks, index, tc='x'): ################################################################################ class Solution(collections.abc.Mapping): - ######################################## - def __init__(self, config, splitGSU: bool, cxxCompiler: str, srcName: str = ""): + ######################################## # need to be sure PSRR is passing to all fxns + def __init__(self, config, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: List[IsaVersion], cxxCompiler: str, srcName: str = ""): self._name = None self.cxxCompiler = cxxCompiler self.srcName = srcName @@ -1074,7 +1074,6 @@ def __init__(self, config, splitGSU: bool, cxxCompiler: str, srcName: str = ""): else: self["InternalSupportParams"] = defaultInternalSupportParams - # assign parameters with defaults for key in defaultSolution: assignParameterWithDefault(self._state, key, config, defaultSolution) @@ -1126,7 +1125,8 @@ def __init__(self, config, splitGSU: bool, cxxCompiler: str, srcName: str = ""): self._state = state break self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None - self.initHelperKernelObjects() + + self.initHelperKernelObjects(supportedISA) # these keys are copied from ProblemType to internal that may be overridden InternalKeys = ["UseSgprForGRO","VectorStore"] @@ -1144,11 +1144,11 @@ def getKernels(self): ######################################## # create Helper Kernels - def initHelperKernelObjects(self): + def initHelperKernelObjects(self, supportedISA: List[IsaVersion]): self.initBetaOnlyKernelObjects() - self.initConversionKernelObjects() + self.initConversionKernelObjects(supportedISA) self.initActivationEnumHeaderObjects() - self.initActivationFunctionObjects() + self.initActivationFunctionObjects(supportedISA) self.initActivationOnlyKernelObjects() self.initReductionKernelObjects() @@ -1178,7 +1178,7 @@ def initBetaOnlyKernelObjects(self): ######################################## # create Conversion Kernels - def initConversionKernelObjects(self): + def initConversionKernelObjects(self, supportedArchs: List[tuple]): self.conversionKernelObjects = [] load_vector_width = [1, 2] if self["ProblemType"]["DataType"].isDouble() else [1, 2, 4] genPGRPostKernels = True @@ -1205,7 +1205,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs)) for btype in typeList: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1218,7 +1218,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs)) else: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1229,7 +1229,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs)) def initActivationEnumHeaderObjects(self): self.activationEnumHeaderObjects = [] @@ -1240,7 +1240,7 @@ def initActivationEnumHeaderObjects(self): state["KernelLanguage"] = "Source" self.activationEnumHeaderObjects.append(KernelWriterActivationEnumHeader(state)) - def initActivationFunctionObjects(self): + def initActivationFunctionObjects(self, supportedISA: List[IsaVersion]): self.activationFunctionObjects = [] if self["ProblemType"]["ActivationType"] in ['all', 'hipblaslt_all']: state = {} @@ -1248,7 +1248,9 @@ def initActivationFunctionObjects(self): state["ProblemType"]["GroupedGemm"] = False state["KernelLanguage"] = "Source" state["Kernel"] = {"WavefrontSize": self["WavefrontSize"], "ISA": tuple(self["ISA"])} - self.activationFunctionObjects.append(KernelWriterActivationFunction(state, self.cxxCompiler)) + if not isinstance(supportedISA, list): + raise Exception(f"{type(supportedISA)}") + self.activationFunctionObjects.append(KernelWriterActivationFunction(state, self.cxxCompiler, supportedISA)) def initActivationOnlyKernelObjects(self): self.activationOnlyKernelObjects = [] @@ -1316,7 +1318,7 @@ def getMIOutputInfo(state): ######################################## # assign tile sizes @staticmethod - def assignProblemIndependentDerivedParameters(state): + def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool=False): if globalParameters["NewClient"] != 2: print("WARNING: Old client deprecated, NewClient parameter being set to 2.") @@ -1330,10 +1332,10 @@ def assignProblemIndependentDerivedParameters(state): state["Valid"] = True if (not state["ProblemType"]["StridedBatched"]) and (not state["ProblemType"]['Batched']): - reject(state, "General Batched GEMM only support Batched Problem") + reject(state, printRejectionReason, "General Batched GEMM only support Batched Problem") if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): - reject(state, "General Batched GEMM only support GEMM OperationType") + reject(state, printRejectionReason, "General Batched GEMM only support GEMM OperationType") Solution.MatrixInstructionToMIParameters(state) EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None @@ -1346,7 +1348,7 @@ def assignProblemIndependentDerivedParameters(state): and ("ThreadTile" in state and len(state["ThreadTile"]) == 2) : EnableMatrixInstruction = False else: - reject(state, "EnableMatrixInstruction undetermined") + reject(state, printRejectionReason, "EnableMatrixInstruction undetermined") if EnableMatrixInstruction == True: state["MatrixInstM"] = state["MIBlock"][0] @@ -1388,7 +1390,7 @@ def assignProblemIndependentDerivedParameters(state): if "SubGroup0" in state and "SubGroup1" in state and "LocalSplitU" in state: state["NumThreads"] = state["SubGroup0"] * state["SubGroup1"] * state["LocalSplitU"] if (state["NumThreads"] % state['WavefrontSize']) != 0: - reject(state, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") + reject(state, printRejectionReason, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") # macro tile sizes if "SubGroup0" in state and "ThreadTile0" in state: @@ -1398,7 +1400,7 @@ def assignProblemIndependentDerivedParameters(state): if "MacroTile" in state: if state["MacroTile0"] != state["MacroTile"][0] \ or state["MacroTile1"] != state["MacroTile"][1]: - reject(state, "MacroTile mismatch") + reject(state, printRejectionReason, "MacroTile mismatch") # tail loop optimization if (tuple(state["ISA"]) != (9, 4, 2)) or \ @@ -1423,12 +1425,12 @@ def assignProblemIndependentDerivedParameters(state): # state[GlobalReadVectorWidth*] # state[NumLoads*] # only used in SolutionStructs, with classic alg @staticmethod - def setGlobalReadVectorWidth(state, tc, totalVectors, grvw): + def setGlobalReadVectorWidth(state, tc, totalVectors, grvw, printRejectionReason: bool=False): validDepthU = True if grvw not in [1,2,4,8,16,32]: validDepthU = False if totalVectors % state["NumThreads"] != 0: - reject(None, "totalVectors%s %u %% NumThreads %u != 0" \ + reject(None, printRejectionReason, "totalVectors%s %u %% NumThreads %u != 0" \ % (tc, totalVectors, state["NumThreads"])) validDepthU = False @@ -1451,7 +1453,7 @@ def setGlobalReadVectorWidth(state, tc, totalVectors, grvw): # state[LSCA] # state[LSPA] @staticmethod - def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, totalElementsPerp, depthU): + def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, totalElementsPerp, depthU, printRejectionReason: bool=False): if state["WaveSeparateGlobalRead%s"%tc]: totalElementsPerp = roundupRatio(totalElementsPerp, state["NumThreads"] // state["WavefrontSize"]) @@ -1480,7 +1482,7 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota foundValid = True break if not foundValid: - reject(state, "%s: No NumLoadsCoalesced=1 found"%tc) + reject(state, printRejectionReason, "%s: No NumLoadsCoalesced=1 found"%tc) return False # nlc = -1 @@ -1496,29 +1498,29 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota foundValid = True break if not foundValid: - reject(state, "%s: No NumLoadsCoalesced=-1 found"%tc) + reject(state, printRejectionReason, "%s: No NumLoadsCoalesced=-1 found"%tc) return False # nlc = other else: if state["NumLoadsCoalesced%s"%tc] > state["NumLoads%s"%tc]: - reject(state, "%s nlc > numLoads"%tc) + reject(state, printRejectionReason, "%s nlc > numLoads"%tc) return False state["NumLoadsPerpendicular%s"%tc] = state["NumLoads%s"%tc] \ // state["NumLoadsCoalesced%s"%tc] if state["NumLoads%s"%tc] % state["NumLoadsCoalesced%s"%tc] != 0: - reject(state, "%s: numLoads %u %% numLoadsCoalesced %u != 0" \ + reject(state, printRejectionReason, "%s: numLoads %u %% numLoadsCoalesced %u != 0" \ % (tc, state["NumLoads%s"%tc], state["NumLoadsCoalesced%s"%tc])) return False if totalVectorsCoalesced % state["NumLoadsCoalesced%s"%tc] != 0 : - reject(state, "%s: totalVectorsCoalesced %u %% numLoadsPara %u != 0" \ + reject(state, printRejectionReason, "%s: totalVectorsCoalesced %u %% numLoadsPara %u != 0" \ % (tc, totalVectorsCoalesced, state["NumLoadsCoalesced%s"%tc])) return False if totalElementsPerp % state["NumLoadsPerpendicular%s"%tc] != 0: - reject(state, "%s: totalElementsPerp %u %% numLoadsPerp %u != 0" \ + reject(state, printRejectionReason, "%s: totalElementsPerp %u %% numLoadsPerp %u != 0" \ % (tc, totalElementsPerp, state["NumLoadsPerpendicular%s"%tc])) return False @@ -1722,7 +1724,7 @@ def setGlobalLoadTileDimFractional(state, tc, depthU): @staticmethod - def MatrixInstructionToMIParameters(state): + def MatrixInstructionToMIParameters(state, printRejectionReason: bool=False): isa = tuple(state["ISA"]) if len(state["MatrixInstruction"]) == 9: mi = state["MatrixInstruction"] @@ -1745,14 +1747,14 @@ def MatrixInstructionToMIParameters(state): state["MatrixInstruction"] in validMFMA["B1k"]: state["MFMA_BF16_1K"] = True else: - reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], miDataType)) + reject(state, printRejectionReason, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], miDataType)) elif globalParameters["AsmCaps"][isa]["HasWMMA"]: if state["MatrixInstruction"] not in validWMMA: - reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) + reject(state, printRejectionReason, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) else: if not (state["ProblemType"]["DataType"].toChar() in validSMFMA and \ state["MatrixInstruction"] in validSMFMA[state["ProblemType"]["DataType"].toChar()]): - reject(state, "Sparse MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) + reject(state, printRejectionReason, "Sparse MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) # set EnableMatrixInstruction state["EnableMatrixInstruction"] = True @@ -1799,14 +1801,14 @@ def MatrixInstructionToMIParameters(state): ############################################## # check and calculate Wave Separate Global Read @staticmethod - def checkAndAssignWaveSeparateGlobalRead(state, tc): + def checkAndAssignWaveSeparateGlobalRead(state, tc, printRejectionReason: bool=False): # check can we use WaveSeparateGlobalRead numOfWaves = state["NumThreads"] // state["WavefrontSize"] if state["WaveSeparateGlobalRead%s"%tc]: if state["ProblemType"]["TLU%s"%tc] and (state["_DepthU%s"%tc] > 0) and (state["_DepthU%s"%tc] % numOfWaves != 0): - reject(state, "didn't support WaveSeparateGlobalRead when DepthU is not multiple of wave %u in TLU%s" % (state["_DepthU%s"%tc], tc)) + reject(state, printRejectionReason, "didn't support WaveSeparateGlobalRead when DepthU is not multiple of wave %u in TLU%s" % (state["_DepthU%s"%tc], tc)) if not state["ProblemType"]["TLU%s"%tc] and (state["MacroTile%s" % tc] % numOfWaves != 0): - reject(state, "didn't support WaveSeparateGlobalRead when MacroTile is not multiple of wave %u in TLU%s" % (state["MacroTile%s"%tc], tc)) + reject(state, printRejectionReason, "didn't support WaveSeparateGlobalRead when MacroTile is not multiple of wave %u in TLU%s" % (state["MacroTile%s"%tc], tc)) ######################################## @@ -1840,19 +1842,19 @@ def isDirectToVgprSupportDataType(state): ######################################## # determine can we use DirectToVgpr @staticmethod - def isDirectToVgprDoable(state, tc): + def isDirectToVgprDoable(state, tc, printRejectionReason: bool=False): MIindex = 0 if tc == 'A' else 1 numBytes = state["ProblemType"]["DataType"].numBytes() numBytesGR = state["ProblemType"]["DataType%s"%tc].numBytes() # With MatrixInstruction only if not state["EnableMatrixInstruction"] : - reject(state, "DirectToVgpr is for MatrixInstruction only") + reject(state, printRejectionReason, "DirectToVgpr is for MatrixInstruction only") return False # disable the following combinations for initial implementation # TODO: enable them if state["LocalSplitU"] != 1 and (not state["ProblemType"]["TLU%c"%tc]): - reject(state, "DirectToVgpr + LSU + TLU=False has not been enabled yet(tentative)") + reject(state, printRejectionReason, "DirectToVgpr + LSU + TLU=False has not been enabled yet(tentative)") return False if state["DirectToVgprA"] and state["DirectToVgprB"]: @@ -1863,34 +1865,34 @@ def isDirectToVgprDoable(state, tc): state["PrefetchLocalRead"] = 0 # So far, DTVA + DTVB does not perform well (waitcnt is not ideal). # Disable it for now (TODO: improve waitcnt and re-enable) - reject(state, "DirectToVgprA + DirectToVgprB disabled") + reject(state, printRejectionReason, "DirectToVgprA + DirectToVgprB disabled") return False # DTV + input type conversion if state["ProblemType"]["DataType%s"%tc] != state["ProblemType"]["DataType"]: if not state["ConvertAfterDS"]: - reject(state, "DirectToVgpr%s + input conversion + ConvertAfterDS=False not supported"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%s + input conversion + ConvertAfterDS=False not supported"%(tc)) return False # check if the DataType can support DirectToVgpr if not Solution.isDirectToVgprSupportDataType(state): - reject(state, "no DirectToVgpr support for this input data type") + reject(state, printRejectionReason, "no DirectToVgpr support for this input data type") return False # Does not work with TLU = False and PrefetchLocalRead = 0 if (not state["ProblemType"]["TLU%c"%tc]) and state["PrefetchLocalRead"] == 0: - reject(state, "DirectToVgpr%c does not supports TLU%c = False and PrefetchLocalRead = 0"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = False and PrefetchLocalRead = 0"%(tc, tc)) return False # Does not work with TLU = False and CGEMM/DGEMM/DGEMM (not supported) if (not state["ProblemType"]["TLU%c"%tc]) and (state["ProblemType"]["DataType"].isDouble() or \ state["ProblemType"]["DataType"].isComplex()): - reject(state, "DirectToVgpr%c does not supports TLU%c = False + S/C/D/ZGEMM"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = False + S/C/D/ZGEMM"%(tc, tc)) return False if numBytesGR * state["GlobalReadVectorWidth%c"%tc] < 4: # no support for DTV + numBytesGR * GlobalReadVectorWidth< 4 - reject(state, "DirectToVgpr%c does not support TLU%c + numByte * GlobalReadVectorWidth%c < 4"%(tc, tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not support TLU%c + numByte * GlobalReadVectorWidth%c < 4"%(tc, tc, tc)) return False if numBytes < 4: @@ -1898,7 +1900,7 @@ def isDirectToVgprDoable(state, tc): if state["ProblemType"]["TLU%c"%tc]: # use pack logic (with v_perm) same as local read (only if VgprForLocalReadPacking is doable) if not Solution.isVgprForLocalReadPackingDoable(state): - reject(state, "Does not meet the requirement for DirectToVgpr%c + TLU%c + numByte < 4"%(tc, tc)) + reject(state, printRejectionReason, "Does not meet the requirement for DirectToVgpr%c + TLU%c + numByte < 4"%(tc, tc)) return False # force ClusterLocalRead=1 for DTV + pack state["ClusterLocalRead"] = 1 @@ -1906,7 +1908,7 @@ def isDirectToVgprDoable(state, tc): # numBytes >= 4 case if state["ProblemType"]["TLU%c"%tc] and state["MIInputPerThread"] > 1: # no support for numBytes >= 4 + MIInputPerThread > 1 - reject(state, "DirectToVgpr%c does not support TLU%c+ numByte >= 4 + MIInputPerThread > 1"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not support TLU%c+ numByte >= 4 + MIInputPerThread > 1"%(tc, tc)) return False # MatrixInstBM,BN check @@ -1914,52 +1916,52 @@ def isDirectToVgprDoable(state, tc): # for B, MatrixInstBM should be 1 # This is to limit the number of Vgpr if tc == 'A' and not (state['MatrixInstBN'] == 1): - reject(state, "MatrixInstBN should be 1 for DirectToVgprA. Current value is %d"%(state['MatrixInstBN'])) + reject(state, printRejectionReason, "MatrixInstBN should be 1 for DirectToVgprA. Current value is %d"%(state['MatrixInstBN'])) return False if tc == 'B' and not (state['MatrixInstBM'] == 1): - reject(state, "MatrixInstBM should be 1 for DirectToVgprB. Current value is %d"%(state['MatrixInstBM'])) + reject(state, printRejectionReason, "MatrixInstBM should be 1 for DirectToVgprB. Current value is %d"%(state['MatrixInstBM'])) return False # Does not work with WaveSeparateGlobalRead if state["WaveSeparateGlobalRead%c"%tc]: - reject(state, "DirectToVgpr%c does not supports WaveSeparateGlobalRead%c"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports WaveSeparateGlobalRead%c"%(tc, tc)) return False # Does not work with TLU + VectorWidth != GlobalReadVectorWidth (VW = 2 + GRVW = 1 or VW = 1 + GRVW = 2 does not work) if state["ProblemType"]["TLU%c"%tc] and state["VectorWidth%s"%tc] != state["GlobalReadVectorWidth%c"%tc]: - reject(state, "DirectToVgpr%c does not supports TLU + VectorWidth%s(=%u) != GlobalReadVectorWidth%c(%u)"%(tc, tc, state["VectorWidth%s"%tc], tc, state["GlobalReadVectorWidth%c"%tc])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU + VectorWidth%s(=%u) != GlobalReadVectorWidth%c(%u)"%(tc, tc, state["VectorWidth%s"%tc], tc, state["GlobalReadVectorWidth%c"%tc])) return False # Does not work with TLU=False and NumLoadsCoalesced != DepthU//(MatrixInstK*GRVW*LSU//MIInputPerThread) if (not state["ProblemType"]["TLU%c"%tc]) and \ state["NumLoadsCoalesced%c"%tc] != state["DepthU"] // (state["MatrixInstK"] * state["GlobalReadVectorWidth%c"%tc] * state["LocalSplitU"] // state["MIInputPerThread"]): - reject(state, "DirectToVgpr%c does not supports TLU=False and NumLoadsCoalesced%c != DepthU//(MatrixInstK*GlobalReadVectorWidth*LocalSplitU//MIInputPerThread(=%u))"%(tc, tc, state["MIInputPerThread"])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU=False and NumLoadsCoalesced%c != DepthU//(MatrixInstK*GlobalReadVectorWidth*LocalSplitU//MIInputPerThread(=%u))"%(tc, tc, state["MIInputPerThread"])) return False # TLU=False case, need GlobalReadVectorWidth == LocalReadVectorWidth if (not state["ProblemType"]["TLU%c"%tc]) and \ state["GlobalReadVectorWidth%c"%tc] != state["LocalReadVectorWidth"]: - reject(state, "DirectToVgpr%c does not supports TLU=False GlobalReadVectorWidth%c(%u) != LocalReadVectorWidth(%u)"%(tc, tc, state["GlobalReadVectorWidth%c"%tc], state["LocalReadVectorWidth"])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU=False GlobalReadVectorWidth%c(%u) != LocalReadVectorWidth(%u)"%(tc, tc, state["GlobalReadVectorWidth%c"%tc], state["LocalReadVectorWidth"])) return False # Does not work with SIA<3 if state["ScheduleIterAlg"] < 3: - reject(state, "DirectToVgpr%c does not supports ScheduleIterAlg < 3"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports ScheduleIterAlg < 3"%(tc)) return False # Does not work with InnerUnroll>1 if state["InnerUnroll"]>1: - reject(state, "DirectToVgpr%c does not supports InnerUnroll>1"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports InnerUnroll>1"%(tc)) return False # Reject TLU = UnrollMajorLDS if state["ProblemType"]["TLU%c"%tc] == state["UnrollMajorLDS%c"%tc]: - reject(state, "DirectToVgpr%c does not supports TLU%c = UnrollMajorLDS%c"%(tc, tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = UnrollMajorLDS%c"%(tc, tc, tc)) return False # does not work with UnrollLoopSwapGlobalReadOrder if state["UnrollLoopSwapGlobalReadOrder"]: - reject(state, "DirectToVgpr%c does not supports UnrollLoopSwapGlobalReadOrder"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports UnrollLoopSwapGlobalReadOrder"%(tc)) return False # does not work with PGR2 + EPS @@ -1969,17 +1971,17 @@ def isDirectToVgprDoable(state, tc): # does not work with Sparse if state["ProblemType"]["Sparse"]: - reject(state, "DirectToVgpr%c does not supports Sparse"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports Sparse"%(tc)) return False # for DTVA/DTVB, does not work with PGR0 if state["PrefetchGlobalRead"] == 0: - reject(state, "DirectToVgpr%c does not supports PrefetchGlobalRead == 0."%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports PrefetchGlobalRead == 0."%(tc)) return False # for DTVA, does not work with NN and TLDS0 if tc == 'A' and state["TransposeLDS"] == 0 and (not state["ProblemType"]["TransposeA"] and not state["ProblemType"]["TransposeB"]): - reject(state, "DirectToVgpr%c does not supports NN case with TransposeLDS == 0."%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports NN case with TransposeLDS == 0."%(tc)) return False # for DTVA, does not work with TT and Tail-loop @@ -2000,7 +2002,7 @@ def isDirectToVgprDoable(state, tc): ######################################## # determine can we use DirectToLds @staticmethod - def isDirectToLdsDoable(state, tc): + def isDirectToLdsDoable(state, tc, printRejectionReason: bool=False): # x2/x4 support for directToLds (no longer supported) # numelements_perlane = 4/numBytes @@ -2021,85 +2023,85 @@ def isDirectToLdsDoable(state, tc): # numBytesPerLoad == 4 only if numBytesPerLoad != 4: - reject(state, "DirectToLds can only be used with buffer loads requiring 1 register") + reject(state, printRejectionReason, "DirectToLds can only be used with buffer loads requiring 1 register") return False # so far MFMA only (TODO: enable non MFMA case) if not state["EnableMatrixInstruction"]: - reject(state, "DirectToLds is for MatrixInstruction only for now (tentative)") + reject(state, printRejectionReason, "DirectToLds is for MatrixInstruction only for now (tentative)") return False # so far, DirectToLds does not work with StreamK (TODO: enable StreamK case) if state["StreamK"]: - reject(state, "DirectToLds does not support StreamK (tentative)") + reject(state, printRejectionReason, "DirectToLds does not support StreamK (tentative)") return False # DTL + LocalReadVectorWidth > MIInputPerThread does not work # Need support for TailLoop if state["LocalReadVectorWidth"] > state["MIInputPerThread"]: - reject(state, "DirectToLds does not work with LocalReadVectorWidth > MIInputPerThread") + reject(state, printRejectionReason, "DirectToLds does not work with LocalReadVectorWidth > MIInputPerThread") return False if state["AssertSummationElementMultiple"] % state["GlobalReadVectorWidth%c"%tc] != 0: - reject(state, "can't use DirectToLds with AssertSummationElementMultiple(%u) %% GlobalReadVectorWidth%c(%u)" % \ + reject(state, printRejectionReason, "can't use DirectToLds with AssertSummationElementMultiple(%u) %% GlobalReadVectorWidth%c(%u)" % \ (state["AssertSummationElementMultiple"], tc, state["GlobalReadVectorWidth%c"%tc])) return False if state["NumThreads"] % state["WavefrontSize"] != 0: - reject(state, "can't use DirectToLds for NumThreads % WavefrontSize != 0") + reject(state, printRejectionReason, "can't use DirectToLds for NumThreads % WavefrontSize != 0") return False if state["ProblemType"]["TLU%c"%tc] == state["UnrollMajorLDS%c" % tc]: - reject(state, "can't use DirectToLds for TLU%c == UnrollMajorLDS%c"%(tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for TLU%c == UnrollMajorLDS%c"%(tc, tc)) return False # avoid picking x2&x4 for precisions < f32/f64 in [ProblemType][TLU] == TRUE if not state["EnableMatrixInstruction"]: if state["GlobalReadVectorWidth%c"%tc] * numBytesAB * state["WavefrontSize"] > 256: - reject(state, "can't use DirectToLds for not EnableMatrixInstruction and GlobalReadVectorWidth%c * bpe%c * WavefrontSize > 256"%(tc,tc)) + reject(state, printRejectionReason, "can't use DirectToLds for not EnableMatrixInstruction and GlobalReadVectorWidth%c * bpe%c * WavefrontSize > 256"%(tc,tc)) return False if state["WaveSeparateGlobalRead%c" % tc]: if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytesAB != state["WavefrontSize"] * state["GlobalReadVectorWidth%c"%tc] * numBytesAB: - reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe!= WavefrontSize * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for LSC%c and LSP%c * bpe!= WavefrontSize * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) return False else: if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytesAB != state["NumThreads"] * state["GlobalReadVectorWidth%c"%tc] * numBytesAB: - reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe != NumThreads * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for LSC%c and LSP%c * bpe != NumThreads * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) return False # so far, DirectToLds does not work well with PGR=2 # performance is not good and a lot of ds_read for DTL can cause scheduling issue(need fix) if state["PrefetchGlobalRead"] == 2: - reject(state, "can't use DirectToLds for PrefetchGlobalRead == 2") + reject(state, printRejectionReason, "can't use DirectToLds for PrefetchGlobalRead == 2") return False # so far, DirectToLds does not work with LRVW=2 if state["LocalReadVectorWidth"] == 2: - reject(state, "can't use DirectToLds for LocalReadVectorWidth == 2") + reject(state, printRejectionReason, "can't use DirectToLds for LocalReadVectorWidth == 2") return False # Does not work with (NumLoadsCoalesced>1 and UseInstOffsetForGRO) + DGEMM if state["ProblemType"]["DataType"].isDouble() and \ (state["NumLoadsCoalesced%c"%tc] > 1 and state["UseInstOffsetForGRO"]): - reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 and UseInstOffsetForGRO for dgemm"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 and UseInstOffsetForGRO for dgemm"%(tc, tc)) return False # Does not work with NumLoadsCoalesced>1 + ZGEMM if state["ProblemType"]["DataType"].isDoubleComplex() and state["NumLoadsCoalesced%c"%tc] > 1: - reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 for zgemm"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 for zgemm"%(tc, tc)) return False # Does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1 (cannot schedule DTL global read after local read) if state["PrefetchGlobalRead"] == 2 and state["PrefetchLocalRead"] == 1: - reject(state, "DirectToLds%c does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1"%(tc)) + reject(state, printRejectionReason, "DirectToLds%c does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1"%(tc)) return False # DirectToLds does not work if MacroTile is not power of 2 # LDS offset swap/rotate logic works only when MacroTile is power of 2 mt = state["MacroTile%c"%tc] if mt & (mt - 1) != 0: - reject(state, "can't use DirectToLds if MacroTile%s is not power of 2"%tc) + reject(state, printRejectionReason, "can't use DirectToLds if MacroTile%s is not power of 2"%tc) return False # DirectToLds does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced < 8 @@ -2108,13 +2110,13 @@ def isDirectToLdsDoable(state, tc): # current offset swap logic does not work if (not state["ProblemType"]["TLU%c"%tc]) and state["ProblemType"]["DataType"].numRegisters() > 1 and \ state["_DepthU%s"%tc] // state["NumLoadsCoalesced%c"%tc] < 8: - reject(state, "DirectToLds%c does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced%c < 8"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced%c < 8"%(tc, tc)) return False return True @staticmethod - def getDivisorName(state, tC): + def getDivisorName(state, tC, printRejectionReason: bool=False): if state["GlobalReadCoalesceGroup{}".format(tC)]: if state["GlobalReadCoalesceVector{}".format(tC)]: divisorName = "LVC{}".format(tC) @@ -2131,7 +2133,7 @@ def getDivisorName(state, tC): ######################################## # assign all derived parameters @staticmethod - def assignDerivedParameters(state, index, depthuList, splitGSU: bool): + def assignDerivedParameters(state, index, depthuList, splitGSU: bool, printRejectionReason: bool=False): state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -2179,38 +2181,38 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["GlobalSplitU"] = 0 # Cannot enable both Stream-K and GSU state["GlobalSplitUAlgorithm"] = "MultipleBuffer" # Set default Algorithm if not (state["ProblemType"]["DataType"].isSingle() or state["ProblemType"]["DataType"].isHalf() or state["ProblemType"]["DataType"].isBFloat16()): - reject(state, "Type {} for DataType not yet supported with StreamK".format(state["ProblemType"]["DataType"].toChar())) + reject(state, printRejectionReason, "Type {} for DataType not yet supported with StreamK".format(state["ProblemType"]["DataType"].toChar())) if not (state["ProblemType"]["DataTypeA"].isSingle() or state["ProblemType"]["DataTypeA"].isHalf() or state["ProblemType"]["DataTypeA"].isBFloat16()): - reject(state, "Type {} for DataTypeA not yet supported with StreamK".format(state["ProblemType"]["DataTypeA"].toChar())) + reject(state, printRejectionReason, "Type {} for DataTypeA not yet supported with StreamK".format(state["ProblemType"]["DataTypeA"].toChar())) if not (state["ProblemType"]["DataTypeB"].isSingle() or state["ProblemType"]["DataTypeB"].isHalf() or state["ProblemType"]["DataTypeB"].isBFloat16()): - reject(state, "Type {} for DataTypeB not yet supported with StreamK".format(state["ProblemType"]["DataTypeB"].toChar())) + reject(state, printRejectionReason, "Type {} for DataTypeB not yet supported with StreamK".format(state["ProblemType"]["DataTypeB"].toChar())) if not (state["ProblemType"]["DestDataType"].isSingle() or state["ProblemType"]["DestDataType"].isHalf() or state["ProblemType"]["DestDataType"].isBFloat16()): - reject(state, "Type {} for DestDataType not yet supported with StreamK".format(state["ProblemType"]["DestDataType"].toChar())) + reject(state, printRejectionReason, "Type {} for DestDataType not yet supported with StreamK".format(state["ProblemType"]["DestDataType"].toChar())) if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: - reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") + reject(state, printRejectionReason, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") if not state["EnableMatrixInstruction"]: - reject(state, "Stream-K requires MatrixInstruction") + reject(state, printRejectionReason, "Stream-K requires MatrixInstruction") if globalParameters["AsmCaps"][isa]["HasWMMA"]: - reject(state, "Stream-K untested with WMMA") + reject(state, printRejectionReason, "Stream-K untested with WMMA") # if state["PersistentKernel"]: - # reject(state, "Cannot enable both Stream-K and PersistentKernel") + # reject(state, printRejectionReason, "Cannot enable both Stream-K and PersistentKernel") if not state["ProblemType"]["StridedBatched"]: - reject(state, "General batch not supported with Stream-K") + reject(state, printRejectionReason, "General batch not supported with Stream-K") if state["ProblemType"]["GroupedGemm"]: - reject(state, "Grouped gemm not yet supported with Stream-K") + reject(state, printRejectionReason, "Grouped gemm not yet supported with Stream-K") if state["ScheduleGlobalRead"] != 1: - reject(state, "ScheduleGlobalRead not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleGlobalRead not supported with Stream-K") if state["ScheduleLocalWrite"] != 1: - reject(state, "ScheduleLocalWrite not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleLocalWrite not supported with Stream-K") if state["ScheduleIterAlg"] != 1 and state["ScheduleIterAlg"] != 3: - reject(state, "ScheduleIterAlg not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleIterAlg not supported with Stream-K") if state["StreamKAtomic"] == 1: if not state["ProblemType"]["DataType"].isSingle(): - reject(state, "Atomic Stream-K currently only tested for SGEMM") + reject(state, printRejectionReason, "Atomic Stream-K currently only tested for SGEMM") if not state["BufferStore"]: - reject(state, "Atomic Stream-K requires BufferStore") + reject(state, printRejectionReason, "Atomic Stream-K requires BufferStore") if state["LocalSplitU"] > 1: - reject(state, "Atomic Stream-K not working with LocalSplitU") + reject(state, printRejectionReason, "Atomic Stream-K not working with LocalSplitU") if not state["Valid"]: return else: @@ -2242,16 +2244,16 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["NonTemporal%s"%ch] -= 4 if state["WavefrontSize"] == 32 and not globalParameters["ArchCaps"][isa]["HasWave32"]: - reject(state, "WavefrontSize=32 not supported for ISA {}".format(isa)) + reject(state, printRejectionReason, "WavefrontSize=32 not supported for ISA {}".format(isa)) return if state["WavefrontSize"] == 32 and state["KernelLanguage"] == "Source": - reject(state, "WavefrontSize=32 not yet supported for source kernels.") + reject(state, printRejectionReason, "WavefrontSize=32 not yet supported for source kernels.") return if state["EnableMatrixInstruction"]: if not (globalParameters["AsmCaps"][isa]["HasMFMA"] or globalParameters["AsmCaps"][isa]["HasWMMA"]): - reject(state, f"isa {isa} doesn't support matrix instruction") + reject(state, printRejectionReason, f"isa {isa} doesn't support matrix instruction") return if not (state["ProblemType"]["DataType"].isSingle() \ or state["ProblemType"]["DataType"].isDouble() \ @@ -2260,62 +2262,62 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): or state["ProblemType"]["DataType"].isComplex() \ or state["ProblemType"]["DataType"].is8bitFloat() \ or state["ProblemType"]["DataType"].isInt8()): - reject(state, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) + reject(state, printRejectionReason, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) return if (not globalParameters["AsmCaps"][isa]["HasMFMA"] and globalParameters["AsmCaps"][isa]["HasWMMA"] and (state["WavefrontSize"] == 64)): print2("!! Warning: WMMA only well tune on WGP mode, wave size = 32") - # reject(state, "WMMA only suppport on WGP mode, wave size = 32") + # reject(state, printRejectionReason, "WMMA only suppport on WGP mode, wave size = 32") # return if not state["MIBlock"] or len(state["MIBlock"]) != 6: - reject(state, "invalid MIBlock") + reject(state, printRejectionReason, "invalid MIBlock") return if not state["MIWaveGroup"] or len(state["MIWaveGroup"]) != 2: - reject(state, "invalid MIWaveGroup") + reject(state, printRejectionReason, "invalid MIWaveGroup") return if not state["MIWaveTile"] or len(state["MIWaveTile"]) != 2: - reject(state, "invalid MIWaveTile") + reject(state, printRejectionReason, "invalid MIWaveTile") return if globalParameters["AsmCaps"][isa]["HasMFMA"]: if not state["ProblemType"]["HighPrecisionAccumulate"] \ and state["ProblemType"]["DataType"].numRegisters() < 1 : - reject(state, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ + reject(state, printRejectionReason, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ " in fp32 (or i32) precision. Please add the following config:" + \ "\n - HighPrecisionAccumulate: True") return if globalParameters["AsmCaps"][isa]["HasWMMA"]: if state["ProblemType"]["DataType"].numRegisters() >=1: - reject(state, "WMMA only support half, bf16 and i8 type") + reject(state, printRejectionReason, "WMMA only support half, bf16 and i8 type") return if state["InterleaveAlpha"]: - reject(state, "Matrix instruction doesn't support InterleaveAlpha") + reject(state, printRejectionReason, "Matrix instruction doesn't support InterleaveAlpha") return if state["ProblemType"]["DataType"].isInt8(): if isa[:2] == (9, 4): if tuple(state["MatrixInstruction"])[:3] in ((32, 32, 8), (16, 16, 16)): - reject(state, "v_mfma_i32_32x32x8 and v_mfma_i32_16x16x16 have been deprecated in gfx94x") + reject(state, printRejectionReason, "v_mfma_i32_32x32x8 and v_mfma_i32_16x16x16 have been deprecated in gfx94x") return if state["ProblemType"]["ComputeDataType"].isDouble(): # See [4,4,4,4] snop for more info if state["MatrixInstruction"] == [4,4,4,4] and (not state['ISA'] == [9,0,10]) and state["ScheduleIterAlg"] == 3: - reject(state, "Currently Matrix instructions [4,4,4,4] is disabled.") + reject(state, printRejectionReason, "Currently Matrix instructions [4,4,4,4] is disabled.") return else: if not state["ProblemType"]["HighPrecisionAccumulate"] \ and state["ProblemType"]["ComputeDataType"].numRegisters() > state["ProblemType"]["DataType"].numRegisters() : - reject(state, "For non-MI Kernel, if sizeof(ComputeDataType) > sizeof(DataType), " + \ + reject(state, printRejectionReason, "For non-MI Kernel, if sizeof(ComputeDataType) > sizeof(DataType), " + \ "Please add the following config:" + \ "\n - HighPrecisionAccumulate: True") return if state["ProblemType"]["Sparse"]: - reject(state, "Sparse A problem is only supported by SMFMA MI kernel.") + reject(state, printRejectionReason, "Sparse A problem is only supported by SMFMA MI kernel.") return if state["ThreadTile0"] > 16 or state["ThreadTile1"] > 16: - reject(state, "Invalid value for ThreadTile") + reject(state, printRejectionReason, "Invalid value for ThreadTile") return if state["ScheduleIterAlg"] == 2 or state["ScheduleIterAlg"] == 3: - reject(state, "SIA2 and SIA3 only support MatrixInstruction") + reject(state, printRejectionReason, "SIA2 and SIA3 only support MatrixInstruction") return if state["ProblemType"]["Tensor0"]==0: @@ -2340,7 +2342,7 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["MIWaveTileB"] = state["MIWaveTile"][0] if state["ProblemType"]["Sparse"] == 2 and state["DirectToVgprSparseMetadata"]: - reject(state, "Sparse B does not supprot DirectToVgprSparseMetadata") + reject(state, printRejectionReason, "Sparse B does not supprot DirectToVgprSparseMetadata") return @@ -2393,7 +2395,7 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): for (tc,batchMask) in (('A', 0x1), ('B', 0x2)): freeDims = [i for i in problemType["IndexAssignments%s"%tc] if i in problemType["IndicesFree"]] if not freeDims: - reject(state, "tensor%s contains no free indices.") + reject(state, printRejectionReason, "tensor%s contains no free indices.") return False # Determine which indices will be packed together as this impacts several different parms (sizes, magic numbers, etc) @@ -2444,11 +2446,11 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["DirectToLds"] = False state["_UseSgprForGRO"] = False if state["PrefetchGlobalRead"] == 2: - reject(state, "BufferLoad=0 does not support PrefetchGlobalRead=2") + reject(state, printRejectionReason, "BufferLoad=0 does not support PrefetchGlobalRead=2") return if problemType["UseBias"]: - reject(state, "BufferLoad=0 does not support UseBias due to no suppress no load.") + reject(state, printRejectionReason, "BufferLoad=0 does not support UseBias due to no suppress no load.") return #These modes only work under certain conditions, apply them here: @@ -2515,15 +2517,15 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): state["VectorWidthMetadata"] = state["VectorWidthA"] if state["ProblemType"]["Sparse"] == 1 else state["VectorWidthB"] # if state["EnableMatrixInstruction"] and not state["SourceSwap"] and (state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1): - # reject(state, "not implement VectorWidth without SourceSwap") + # reject(state, printRejectionReason, "not implement VectorWidth without SourceSwap") # TT0,1 both must be multiples of VW, b/c of rC, rA, rB if state["EnableMatrixInstruction"]: if (state["MIWaveTile"][0] % state["VectorWidthA"]) != 0: - reject(state, "MIWaveTile0(%u) should be multiple of VectorWidthA(%u)" % (state["MIWaveTile"][0], state["VectorWidthA"])) + reject(state, printRejectionReason, "MIWaveTile0(%u) should be multiple of VectorWidthA(%u)" % (state["MIWaveTile"][0], state["VectorWidthA"])) return if (state["MIWaveTile"][1] % state["VectorWidthB"]) != 0: - reject(state, "MIWaveTile0(%u) should be multiple of VectorWidthB(%u)" % (state["MIWaveTile"][1], state["VectorWidthB"])) + reject(state, printRejectionReason, "MIWaveTile0(%u) should be multiple of VectorWidthB(%u)" % (state["MIWaveTile"][1], state["VectorWidthB"])) return if len(problemType["IndicesSummation"]) > 1: @@ -2538,15 +2540,15 @@ def assignDerivedParameters(state, index, depthuList, splitGSU: bool): if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: if not state["ProblemType"]["HighPrecisionAccumulate"] and state["AssertFree0ElementMultiple"] % 2 != 0: # beta-on-edge has AF0EM requirement except for HPA kernels - reject(state, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") + reject(state, printRejectionReason, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") return if state["ConvertAfterDS"]: if (state["ProblemType"]["DataType"].isHalf() == False): - reject(state, "ConvertAfterDS only support DataType half") + reject(state, printRejectionReason, "ConvertAfterDS only support DataType half") return if (state["ProblemType"]["DataTypeA"].isAnyFloat8() == False) and (state["ProblemType"]["DataTypeB"].isAnyFloat8() == False): - reject(state, "one of DataTypeA or DataTypeB need to be float8") + reject(state, printRejectionReason, "one of DataTypeA or DataTypeB need to be float8") return def calcLdsPad(lrvw: int) -> int: @@ -2562,7 +2564,7 @@ def calcLdsPad(lrvw: int) -> int: optPadA //= 2 readRegsA //= 2 if (not globalParameters["AsmCaps"][isa]['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): - reject(state, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") + reject(state, printRejectionReason, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") return if state["EnableMatrixInstruction"]: # for readRegs = 1 or 4, we need to double pad for MI16x16xNx1 to avoid bank conflict. @@ -2787,7 +2789,7 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo except ValueError: # i.e., StaggerUStride == 0 staggerStrideShift = 0 if staggerStrideShift < 0: - reject(state, "StaggerUStride=%u is less than size of DepthU=%u * BytesPerElement=%u" \ + reject(state, printRejectionReason, "StaggerUStride=%u is less than size of DepthU=%u * BytesPerElement=%u" \ % (state["StaggerUStride"], state["DepthU"], bpeAB)) #print "staggerStrideShift=", staggerStrideShift, "depthu=", state["DepthU"] state["_staggerStrideShift"] = staggerStrideShift @@ -2803,9 +2805,9 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo state["LocalReadVectorWidth"] = state["MIInputPerThread"] else: if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: - reject(state, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) + reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) if state["LocalReadVectorWidth"] > state["MIInputPerThread"] and not state["TransposeLDS"]: - reject(state, "LocalReadVectorWidth require Transpose LDS") + reject(state, printRejectionReason, "LocalReadVectorWidth require Transpose LDS") if autoLRVW: if state["LocalReadVectorWidth"] // state["MIInputPerThread"] > 1: @@ -2842,7 +2844,7 @@ def calSwizzleK(state, tc): if state["MatrixInstBM"] == 1 and state["MIWaveTile"][0] == 1 and state["MIWaveGroup"][0] == 1 and state["ProblemType"]["TLUA"]: state["GlobalReadVectorWidthA"] = 1 else: - reject(state, "GRVWA=-2 is set for skinny MT") + reject(state, printRejectionReason, "GRVWA=-2 is set for skinny MT") elif state["GlobalReadVectorWidthA"] == -1: if state["ProblemType"]["SwizzleTensorA"]: state["GlobalReadVectorWidthA"] = state["MIInputPerThreadA"] * calSwizzleK(state, "A") @@ -2863,7 +2865,7 @@ def calSwizzleK(state, tc): if state["MatrixInstBN"] == 1 and state["MIWaveTile"][1] == 1 and state["MIWaveGroup"][1] == 1 and state["ProblemType"]["TLUB"]: state["GlobalReadVectorWidthB"] = 1 else: - reject(state, "GRVWB=-2 is set for skinny MT") + reject(state, printRejectionReason, "GRVWB=-2 is set for skinny MT") elif state["GlobalReadVectorWidthB"] == -1: if state["ProblemType"]["SwizzleTensorB"]: state["GlobalReadVectorWidthB"] = state["MIInputPerThreadB"] * calSwizzleK(state, "B") @@ -2880,34 +2882,34 @@ def calSwizzleK(state, tc): for tc in ("A", "B",): if state["ProblemType"][f"SwizzleTensor{tc}"]: if not state["EnableMatrixInstruction"]: - reject(state, f"Tensor {tc} swizzling supports MI only") + reject(state, printRejectionReason, f"Tensor {tc} swizzling supports MI only") # Print rejection reason instead of force set # 16 means bytes of buffer_load_dwordx4 SwizzlePackK = calSwizzleK(state, tc) if state[f"GlobalReadVectorWidth{tc}"] != state[f"MIInputPerThread{tc}"] * SwizzlePackK: GRVW_TC = state[f"GlobalReadVectorWidth{tc}"] MIInPerThread = state[f"MIInputPerThread{tc}"] - reject(state, f"SwizzleTensor{tc} doesn't support GRVW{tc} ({GRVW_TC}) != MIInputPerThread{tc} ({MIInPerThread}) * {SwizzlePackK}") + reject(state, printRejectionReason, f"SwizzleTensor{tc} doesn't support GRVW{tc} ({GRVW_TC}) != MIInputPerThread{tc} ({MIInPerThread}) * {SwizzlePackK}") # TODO- increasing VW might have better perf. But it'll change the swizzling pattern. if state[f"VectorWidth{tc}"] != 1: VW_TC = state[f"VectorWidth{tc}"] - reject(state, f"SwizzleTensor{tc} requires VectorWidth{tc} ({VW_TC}) == 1") + reject(state, printRejectionReason, f"SwizzleTensor{tc} requires VectorWidth{tc} ({VW_TC}) == 1") if state["ProblemType"]["SwizzleTensorA"]: if not state["DirectToVgprA"]: - reject(state, f"Tensor A swizzling requires DirectToVgprA") + reject(state, printRejectionReason, f"Tensor A swizzling requires DirectToVgprA") if not state["ProblemType"]["TransposeA"]: - reject(state, f"Tensor A swizzling supports TN or TT only") + reject(state, printRejectionReason, f"Tensor A swizzling supports TN or TT only") if state["ProblemType"]["SwizzleTensorB"]: if not state["DirectToVgprB"]: - reject(state, f"Tensor B swizzling requires DirectToVgprB") + reject(state, printRejectionReason, f"Tensor B swizzling requires DirectToVgprB") if state["ProblemType"]["TransposeB"]: - reject(state, f"Tensor B swizzling supports TN or NN only") + reject(state, printRejectionReason, f"Tensor B swizzling supports TN or NN only") # TODO- NN fails validation due to DTVB + Tail-Loop is not working correctly if not (state["ProblemType"]["TransposeA"] and not state["ProblemType"]["TransposeB"]): - reject(state, f"Tensor B swizzling supports TN only") + reject(state, printRejectionReason, f"Tensor B swizzling supports TN only") # Force GRVW the same when UnrollLoopSwapGlobalReadOrder = 1. if genGRVWA and state["UnrollLoopSwapGlobalReadOrder"] == 1: @@ -2917,15 +2919,15 @@ def calSwizzleK(state, tc): # reject - VW too big if (state["VectorWidthA"] * state["ProblemType"]["DataType"].numBytes()) > 16: - reject(state, "VWA * DataType.numBytes() > 16") + reject(state, printRejectionReason, "VWA * DataType.numBytes() > 16") if (state["VectorWidthB"] * state["ProblemType"]["DataType"].numBytes()) > 16: - reject(state, "VWB * DataType.numBytes() > 16") + reject(state, printRejectionReason, "VWB * DataType.numBytes() > 16") # reject - GRVW too big if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: - reject(state, "GRVWA * DataTypeA.numBytes() > 16") + reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") if (state["GlobalReadVectorWidthB"] * state["ProblemType"]["DataTypeB"].numBytes()) > 16: - reject(state, "GRVWB * DataTypeB.numBytes() > 16") + reject(state, printRejectionReason, "GRVWB * DataTypeB.numBytes() > 16") ######################################## # Search DepthU @@ -3020,7 +3022,7 @@ def calSwizzleK(state, tc): if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: if state["ProblemType"]["DataType"].numRegisters() == 0.5 and (not state["ProblemType"]["HighPrecisionAccumulate"]): if state["GlobalReadVectorWidthA"] == 1 or state["GlobalReadVectorWidthB"] == 1: - reject(state, "HalfEcc requires HPA if glvw = 1") + reject(state, printRejectionReason, "HalfEcc requires HPA if glvw = 1") break if state["ProblemType"]["Sparse"] and not state["DirectToVgprSparseMetadata"]: @@ -3030,13 +3032,13 @@ def calSwizzleK(state, tc): grvw = state["GlobalReadVectorWidthB"] // 4 vw = state["VectorWidthB"] // 4 if state["GlobalReadVectorWidthB"] % 4 != 0: - reject(state, "Sparse B requires GRVWB %% 4 == 0, current GRVWB is %u"%state["GlobalReadVectorWidthB"]) + reject(state, printRejectionReason, "Sparse B requires GRVWB %% 4 == 0, current GRVWB is %u"%state["GlobalReadVectorWidthB"]) break else: grvw = state["GlobalReadVectorWidthA"] // 4 vw = state["VectorWidthA"] // 4 if state["GlobalReadVectorWidthA"] % 4 != 0: - reject(state, "Sparse A requires GRVWA %% 4 == 0, current GRVWA is %u"%state["GlobalReadVectorWidthA"]) + reject(state, printRejectionReason, "Sparse A requires GRVWA %% 4 == 0, current GRVWA is %u"%state["GlobalReadVectorWidthA"]) break @@ -3070,7 +3072,7 @@ def calSwizzleK(state, tc): if state["ProblemType"]["Sparse"] and state["DirectToVgprSparseMetadata"]: if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1 : - reject(state, "Not implement DTVSM with VW>1") + reject(state, printRejectionReason, "Not implement DTVSM with VW>1") break # Now convert elements to vectors based on GlobalReadVectorWidth @@ -3099,7 +3101,7 @@ def calSwizzleK(state, tc): # this depthU not valid else: - reject(state, "No valid DepthU found") + reject(state, printRejectionReason, "No valid DepthU found") break ######################################## # end DepthU loop @@ -3147,25 +3149,25 @@ def calSwizzleK(state, tc): if state["EnableMatrixInstruction"]: if state["SourceSwap"]: if ((state["VectorWidthA"] % state["StoreVectorWidth"]) != 0): - reject(state, "MFMA SourceSwap mode doesn't support vwA(%u) with svw(%u)" % (state["VectorWidthA"], state["StoreVectorWidth"])) + reject(state, printRejectionReason, "MFMA SourceSwap mode doesn't support vwA(%u) with svw(%u)" % (state["VectorWidthA"], state["StoreVectorWidth"])) return else: if (((state["VectorWidthA"] * state["MIOutputVectorWidth"]) % state["StoreVectorWidth"]) != 0): - reject(state, "MFMA non-SourceSwap mode doesn't support miovw(%u) with svw(%u)" % (state["VectorWidthA"]*state["MIOutputVectorWidth"], state["StoreVectorWidth"])) + reject(state, printRejectionReason, "MFMA non-SourceSwap mode doesn't support miovw(%u) with svw(%u)" % (state["VectorWidthA"]*state["MIOutputVectorWidth"], state["StoreVectorWidth"])) return # LocalSplitU too large? numElementsPerWorkGroup = state["MacroTile0"]*state["MacroTile1"] if numElementsPerWorkGroup < state["NumThreads"]: - reject(state, "NumElementsPerWorkGroup %u < NumThreads %u; reduce LocalSplitU" \ + reject(state, printRejectionReason, "NumElementsPerWorkGroup %u < NumThreads %u; reduce LocalSplitU" \ % (numElementsPerWorkGroup, state["NumThreads"])) return state["NumElementsPerThread"] = numElementsPerWorkGroup // state["NumThreads"] state["GlobalWriteVectorWidth"] = min(state["VectorWidthA"], state["NumElementsPerThread"] ) if state["NumElementsPerThread"] % state["GlobalWriteVectorWidth"] != 0: - reject(state, "LSU NumElementsPerThread %u not divisible into GWVW %u" \ + reject(state, printRejectionReason, "LSU NumElementsPerThread %u not divisible into GWVW %u" \ % (state["NumElementsPerThread"], state["GlobalWriteVectorWidth"])) return state["NumGlobalWriteVectorsPerThread"] = state["NumElementsPerThread"] \ @@ -3175,21 +3177,21 @@ def calSwizzleK(state, tc): # LocalSplitU but can't NumThreads%MacroTile doesn't support sideways store if state["LocalSplitU"] > 1: if not state["SourceSwap"] and state["StoreVectorWidth"] > state["VectorWidthA"]: - reject(state, "LSU and non-SourceSwap doesn't support StoreVectorWidth(%u)>VWA(%u)." \ + reject(state, printRejectionReason, "LSU and non-SourceSwap doesn't support StoreVectorWidth(%u)>VWA(%u)." \ % (state["StoreVectorWidth"], state["VectorWidthA"])) return if not (state["ProblemType"]["ComputeDataType"].isSingle() or state["ProblemType"]["ComputeDataType"].isInt32()): - reject(state, "TODO: LSU doesn't support ComputeDataType!=(single or Int32).") + reject(state, printRejectionReason, "TODO: LSU doesn't support ComputeDataType!=(single or Int32).") return if state["StoreRemapVectorWidth"] > 0: - reject(state, "TODO: LSU doesn't support StoreRemapVectorWidth>0.") + reject(state, printRejectionReason, "TODO: LSU doesn't support StoreRemapVectorWidth>0.") return if state["NumThreads"] % state["MacroTile0"] != 0: - reject(state, "LocalSplitU but NumThreads=%u not divisible by MT0=%u for sideways store" \ + reject(state, printRejectionReason, "LocalSplitU but NumThreads=%u not divisible by MT0=%u for sideways store" \ % (state["NumThreads"], state["MacroTile0"])) return if state["MacroTile0"]*state["MacroTile1"] % state["NumThreads"] != 0: - reject(state, "LocalSplitU but MT0*MT1=%u elements doesn't divide into NumThreads=%u" \ + reject(state, printRejectionReason, "LocalSplitU but MT0*MT1=%u elements doesn't divide into NumThreads=%u" \ % (state["MacroTile0"]*state["MacroTile1"], state["NumThreads"])) return @@ -3205,16 +3207,16 @@ def calSwizzleK(state, tc): (state["_GlobalAccumulation"]) ) if not supported: - reject(state, "GlobalSplitU only compatible with single or asm and (half or mixed) precision") + reject(state, printRejectionReason, "GlobalSplitU only compatible with single or asm and (half or mixed) precision") return if state["ProblemType"]["DataType"].isHalf() and state["KernelLanguage"] == "Assembly": if state["GlobalSplitU"] > 1 and (not state["_GlobalAccumulation"]): if state["AssertFree0ElementMultiple"] < 2: - reject(state, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") + reject(state, printRejectionReason, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]['HasWMMA']: - reject(state, "Half WMMA doesn't support single buffer GSU") + reject(state, printRejectionReason, "Half WMMA doesn't support single buffer GSU") return if state["ProblemType"]["Sparse"] and not state["DirectToVgprSparseMetadata"]: @@ -3271,16 +3273,16 @@ def calSwizzleK(state, tc): # TODO if (0 and state["LSCA"] % state["GlobalReadVectorWidthA"] != 0): - reject(state, "lsca % grvw != 0") + reject(state, printRejectionReason, "lsca % grvw != 0") return if (0 and state["LSPA"] % state["GlobalReadVectorWidthA"] != 0): - reject(state, "lspa % grvw != 0") + reject(state, printRejectionReason, "lspa % grvw != 0") return if (0 and state["LSCB"] % state["GlobalReadVectorWidthB"] != 0): - reject(state, "lscb % grvw != 0") + reject(state, printRejectionReason, "lscb % grvw != 0") return if (0 and state["LSPB"] % state["GlobalReadVectorWidthB"] != 0): - reject(state, "lspb % grvw != 0") + reject(state, printRejectionReason, "lspb % grvw != 0") return state["LVCA"] = roundupRatio(state["LSCA"] , state["GlobalReadVectorWidthA"]) @@ -3306,7 +3308,7 @@ def calSwizzleK(state, tc): # lds buffer size for A, B if state["KernelLanguage"] == "Source" and \ state["LdsPadA"] != state["LdsPadB"]: - reject(state, "Source KernelLanguage only supports LdsPadA == LdsPadB") + reject(state, printRejectionReason, "Source KernelLanguage only supports LdsPadA == LdsPadB") return # NoTailLoop parameter initialization. @@ -3351,23 +3353,23 @@ def calSwizzleK(state, tc): if state["LdsBlockSizePerPadA"]: if state["UnrollMajorLDSA"]: if state["LdsBlockSizePerPadA"] % (state["_DepthUA"] * state["ProblemType"]["DataTypeA"].numBytes()) != 0: - reject(state, "reject: LdsBlockSizePerPadA %u %% depthU %u x bpeA != 0" % (state["LdsBlockSizePerPadA"],state["_DepthUA"])) + reject(state, printRejectionReason, "reject: LdsBlockSizePerPadA %u %% depthU %u x bpeA != 0" % (state["LdsBlockSizePerPadA"],state["_DepthUA"])) if (state["LdsBlockSizePerPadA"] // (state["_DepthUA"] * state["ProblemType"]["DataType"].numBytes())) % state["LSPA"] != 0 and \ state["LSPA"] % (state["LdsBlockSizePerPadA"] // (state["_DepthUA"] * state["ProblemType"]["DataType"].numBytes())) != 0: - reject(state, "can't pad by addrVgpr or instOffset") + reject(state, printRejectionReason, "can't pad by addrVgpr or instOffset") if state["LdsBlockSizePerPadB"]: if state["UnrollMajorLDSB"]: if state["LdsBlockSizePerPadB"] % state["_DepthUB"] * state["ProblemType"]["DataTypeB"].numBytes() != 0: - reject(state, "reject: LdsBlockSizePerPadB %u %% depthU %u x bpeB != 0" % (state["LdsBlockSizePerPadB"],state["_DepthUB"])) + reject(state, printRejectionReason, "reject: LdsBlockSizePerPadB %u %% depthU %u x bpeB != 0" % (state["LdsBlockSizePerPadB"],state["_DepthUB"])) if (state["LdsBlockSizePerPadB"] // (state["_DepthUB"] * state["ProblemType"]["DataType"].numBytes())) % state["LSPB"] != 0 and \ state["LSPB"] % (state["LdsBlockSizePerPadB"] // (state["_DepthUB"] * state["ProblemType"]["DataType"].numBytes())) != 0: - reject(state, "can't pad by addrVgpr or instOffset") + reject(state, printRejectionReason, "can't pad by addrVgpr or instOffset") else: if state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]: - reject(state, "didn't support UnrollMajorLDS in VALU mode yet") + reject(state, printRejectionReason, "didn't support UnrollMajorLDS in VALU mode yet") if state["LdsBlockSizePerPadA"] != 0 or state["LdsBlockSizePerPadB"] != 0: - reject(state, "didn't support LdsBlockSizePerPad in VALU mode yet") + reject(state, printRejectionReason, "didn't support LdsBlockSizePerPad in VALU mode yet") def checkLdsBlockSizePerPad(tc): """ @@ -3455,7 +3457,7 @@ def findValidWriteBlockWidth(nwcv, bpe, bpr): blockWidth = bw break if blockWidth == 0: - reject(state, "invalid local write block width") + reject(state, printRejectionReason, "invalid local write block width") return blockWidth @@ -3517,7 +3519,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): printWarning("Padded address is inconisstent, set LdsBlockSizePerPad%s=0."%tc) state["LdsBlockSizePerPad%s"%tc] = 0 else: - reject(state, "%s's padded address is inconisstent"%tc) + reject(state, printRejectionReason, "%s's padded address is inconisstent"%tc) if(not (state["CustomKernelName"] and state["CustomKernelName"] != "")): #don't check the custom kernel. checkLdsBlockSizePerPad("A") @@ -3554,10 +3556,10 @@ def subCheckLdsBlockSizePerPad(tc, idx): # Re-check DTV + WaveGroup after DTL is confirmed if state["DirectToLds"]: if state["DirectToVgprA"] and state['MIWaveGroup'][1] > 1: - reject(state, "DirectToLds + (DirectToVgprA + WaveGroups along N-Dim) is not supported yet") + reject(state, printRejectionReason, "DirectToLds + (DirectToVgprA + WaveGroups along N-Dim) is not supported yet") return False if state["DirectToVgprB"] and state['MIWaveGroup'][0] > 1: - reject(state, "DirectToLds + (DirectToVgprB + WaveGroups along M-Dim) is not supported yet") + reject(state, printRejectionReason, "DirectToLds + (DirectToVgprB + WaveGroups along M-Dim) is not supported yet") return False # set NoLdsWriteCode if (DirectToVgpr or DirectToLds)A+B is enabled @@ -3579,7 +3581,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): assert(state["LdsPadB"] >= 0) if (state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]) and (not state["EnableMatrixInstruction"]): - reject(state, "UnrollMajorLDS Supports only in EnableMatrixInstruction=1") + reject(state, printRejectionReason, "UnrollMajorLDS Supports only in EnableMatrixInstruction=1") ldsNumBytesA, ldsNumBytesAlignedA, ldsNumBytesB, ldsNumBytesAlignedB, ldsNumBytesMetadata, ldsNumBytesAlignedMetadata = calcLdsNumBytes(state["LdsPadA"], state["LdsBlockSizePerPadA"], state["LdsPadB"], state["LdsBlockSizePerPadB"]) @@ -3627,7 +3629,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): if state["EnableMatrixInstruction"]: if state["DirectToLds"] and state["1LDSBuffer"]: - reject(state, "1LDSBuffer must be 0 for directToLds") + reject(state, printRejectionReason, "1LDSBuffer must be 0 for directToLds") if state["1LDSBuffer"] == -1: if ldsNumBytesAB <= max(ldsSizeOccupancy,32768) or \ @@ -3639,10 +3641,10 @@ def subCheckLdsBlockSizePerPad(tc, idx): if state["1LDSBuffer"]: if not state["PrefetchGlobalRead"]: - reject(state, "PGR=0 already use 1 LDS buffer only") + reject(state, printRejectionReason, "PGR=0 already use 1 LDS buffer only") # Should be able to support as long as NO scheduleLocalWrite if (not state["ScheduleIterAlg"] == 2) and (not state["ScheduleIterAlg"] == 3) and (state["ScheduleLocalWrite"]): - reject(state, "1LDSBuffer only support SIA2 or SIA3, or SIA1 without SLW") + reject(state, printRejectionReason, "1LDSBuffer only support SIA2 or SIA3, or SIA1 without SLW") state["LdsOffsetB"] = ldsNumBytesAlignedA state["LdsOffsetMetadata"] = state["LdsOffsetB"] + ldsNumBytesAlignedB ldsNumBytesAB = ldsNumBytesAlignedA + ldsNumBytesAlignedB + ldsNumBytesMetadata @@ -3684,9 +3686,9 @@ def subCheckLdsBlockSizePerPad(tc, idx): if not state["SourceSwap"]: if not state["StoreRemapVectorWidth"]: - reject(state, "reject to reduce number of kernels") + reject(state, printRejectionReason, "reject to reduce number of kernels") elif state["VectorWidthA"] > 1: - reject(state, "reject to reduce number of kernels") + reject(state, printRejectionReason, "reject to reduce number of kernels") # GuaranteeNoPartial if state["ProblemType"]["TLUA"]: @@ -3704,27 +3706,27 @@ def subCheckLdsBlockSizePerPad(tc, idx): # SourceSwap if state["StoreRemapVectorWidth"]: if state["SourceSwap"]: - reject(state, "SourceSwap not compatible with StoreRemap") + reject(state, printRejectionReason, "SourceSwap not compatible with StoreRemap") return if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1: - reject(state, "VW>1 not compatible with StoreRemap") + reject(state, printRejectionReason, "VW>1 not compatible with StoreRemap") return # Sparse problem if state["ProblemType"]["Sparse"]: if state["PrefetchGlobalRead"] and not state["ExpandPointerSwap"]: - reject(state, "Sparse A kernel only support PGR with EPS=1.") + reject(state, printRejectionReason, "Sparse A kernel only support PGR with EPS=1.") return if state["EnableMatrixInstruction"] and state["MIArchVgpr"]: - reject(state, "Sparse A kernel does not support MIArchVgpr yet.") + reject(state, printRejectionReason, "Sparse A kernel does not support MIArchVgpr yet.") return # Not Support Feature if state["ProblemType"]["Sparse"] == 1 and state["SourceSwap"] : - reject(state, "Sparse A kernel cannot support SourceSwap.") + reject(state, printRejectionReason, "Sparse A kernel cannot support SourceSwap.") return else: if state["ProblemType"]["Sparse"] == 2 and not state["SourceSwap"]: - reject(state, "Sparse B kernel must enable SourceSwap.") + reject(state, printRejectionReason, "Sparse B kernel must enable SourceSwap.") return state["AssertSummationElementMultiple"] = 8 @@ -3741,7 +3743,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): if state["MIArchVgpr"]: if not state["EnableMatrixInstruction"]: - reject(state, "MIArchVgpr only support for MatrixInstruction") + reject(state, printRejectionReason, "MIArchVgpr only support for MatrixInstruction") return if globalParameters["AsmCaps"][isa]["HasMFMA"]: @@ -3750,27 +3752,27 @@ def subCheckLdsBlockSizePerPad(tc, idx): (state["ProblemType"]["ComputeDataType"].isHalf() and state["ProblemType"]["HighPrecisionAccumulate"]) or \ state["ProblemType"]["ComputeDataType"].isInt32() or \ state["ProblemType"]["ComputeDataType"].isComplex()): - reject(state, "MIArchVgpr now only support fp64, fp64c, fp32, fp32c, fp16, int8 MatrixInstruction.") + reject(state, printRejectionReason, "MIArchVgpr now only support fp64, fp64c, fp32, fp32c, fp16, int8 MatrixInstruction.") return #check not support cases and calculate lds resources ldsNumBytesRemapC = 0 if state["StoreRemapVectorWidth"]: if not state["EnableMatrixInstruction"]: - reject(state, "storeRemap only support MatrixInstruction kernel") + reject(state, printRejectionReason, "storeRemap only support MatrixInstruction kernel") return if ((state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer' or state["_GlobalAccumulation"] == 'MultipleBufferSingleKernel')) or \ (state["GlobalSplitU"] == 1 and state["_GlobalAccumulation"] == 'SingleBuffer'): - reject(state, "storeRemap doesn't support GlobalSplitU yet, except GSU algorithm 2") + reject(state, printRejectionReason, "storeRemap doesn't support GlobalSplitU yet, except GSU algorithm 2") return if packedC0 or packedC1: - reject(state, "storeRemap doesn't support packedC0 and packedC1 yet") + reject(state, printRejectionReason, "storeRemap doesn't support packedC0 and packedC1 yet") return if state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4: - reject(state, "storeRemap doesn't support MI4x4 multi blocks in N direction yet") + reject(state, printRejectionReason, "storeRemap doesn't support MI4x4 multi blocks in N direction yet") return if not math.log(state["MacroTile0"],2).is_integer(): - reject(state, "storeRemap only supports power-of-2 MT0") + reject(state, printRejectionReason, "storeRemap only supports power-of-2 MT0") # TODO - this return should be here, but this is a hotfix, # Somehow we have a "Validation Failed" kernel in rocBLAS now (SRVW=4 and MT0=96) and this will stop the whole building process # Actions: 1. Hotfix, comment out this "return" temporarily for that invalidated kernel @@ -3789,15 +3791,15 @@ def subCheckLdsBlockSizePerPad(tc, idx): while srMaxVw < state["StoreRemapVectorWidth"]: state["StoreRemapVectorWidth"] = state["StoreRemapVectorWidth"] // 2 if srMinVw > state["StoreRemapVectorWidth"] or srMaxVw < state["StoreRemapVectorWidth"]: - reject(state, "StoreRemapVectorWidth %u is not allowed for this data type" % state["StoreRemapVectorWidth"]) + reject(state, printRejectionReason, "StoreRemapVectorWidth %u is not allowed for this data type" % state["StoreRemapVectorWidth"]) return if state["StoreRemapVectorWidth"] * state["WavefrontSize"] < state["MacroTile0"]: - reject(state, "storeRemap: Per wave single global write instruction doesn't enough to write one M column." + \ + reject(state, printRejectionReason, "storeRemap: Per wave single global write instruction doesn't enough to write one M column." + \ " Please use larger StoreRemapVectorWidth.") return if (state["MacroTile0"]*state["MatrixInstN"])//state["MIWaveGroup"][0] < state["StoreRemapVectorWidth"]*state["WavefrontSize"]: - reject(state, "storeRemap: number elements of lds less than per wave per local read elements." + \ + reject(state, printRejectionReason, "storeRemap: number elements of lds less than per wave per local read elements." + \ " Please use smaller StoreRemapVectorWidth.") return ldsRemapPad = max(state["StoreRemapVectorWidth"],state["MIOutputVectorWidth"]) @@ -3826,7 +3828,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): # TODO- Remove this DataType test condition, # Currently we do this test is just because we don't want to affect existing logic in rocBLAS if state["ProblemType"]["DataType"].isInt8(): - reject(state, "LDS usage is bound be StoreRemap, thus 1LDSBuffer wouldn't have any help. Skip.") + reject(state, printRejectionReason, "LDS usage is bound be StoreRemap, thus 1LDSBuffer wouldn't have any help. Skip.") return ldsNumBytes = max(ldsNumBytes, ldsNumBytesRemapC) @@ -3915,7 +3917,7 @@ def calcEpilogueTurns(factorDims: List) -> int: state["LdsNumBytes"] = ldsNumBytes ldsSize = ldsNumBytes if ldsSize > globalParameters["MaxLDS"]: - reject(state, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, globalParameters["MaxLDS"])) + reject(state, printRejectionReason, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, globalParameters["MaxLDS"])) return # LoopUnroll = DepthU / LocalSplitU @@ -3924,7 +3926,7 @@ def calcEpilogueTurns(factorDims: List) -> int: if state["LoopUnroll"] * state["LocalSplitU"] != state["DepthU"]: state["Valid"] = False if state["KernelLanguage"] != "Assembly" and state["InnerUnroll"] != 1: - reject(state, "InnerUnroll only supported on assembly") + reject(state, printRejectionReason, "InnerUnroll only supported on assembly") state["LoopUnroll"] //= state["InnerUnroll"] if 0: @@ -3941,7 +3943,7 @@ def calcEpilogueTurns(factorDims: List) -> int: state["LoopIters"] //= state["MatrixInstK"] if state["LoopIters"] < 1: - reject(state, "LoopIters need to greater than 0") + reject(state, printRejectionReason, "LoopIters need to greater than 0") return # Since we use PLR >= LoopIters for allocating numberOfIters vgprBuffer for a while @@ -3950,7 +3952,7 @@ def calcEpilogueTurns(factorDims: List) -> int: # 1 or 2 Byte input + DTVA or DTVB case, does not work with PLR=0. Reject it here. if state["ProblemType"]["DataType"].numBytes() < 4 and \ (state["ProblemType"]["TLUA"] and state["DirectToVgprA"] or state["ProblemType"]["TLUB"] and state["DirectToVgprB"]): - reject(state, "DirectToVgpr does not work with 1 or 2 Byte input + TLU + PrefetchLocalRead(%u) >= LoopIters(%u)"%(state["PrefetchLocalRead"], state["LoopIters"])) + reject(state, printRejectionReason, "DirectToVgpr does not work with 1 or 2 Byte input + TLU + PrefetchLocalRead(%u) >= LoopIters(%u)"%(state["PrefetchLocalRead"], state["LoopIters"])) return state["ClusterLocalRead"] = 0 state["PrefetchLocalRead"] = 0 @@ -3962,31 +3964,31 @@ def calcEpilogueTurns(factorDims: List) -> int: # Multiple = WLR-size / input-size = how many iters could be covered by one WLR ? wlrMultiple = state["LocalReadVectorWidth"]//state["MIInputPerThread"] if wlrMultiple == 0: - reject(state, "LocalReadVectorWidth %u is less than MIInput" % (state["LocalReadVectorWidth"])) + reject(state, printRejectionReason, "LocalReadVectorWidth %u is less than MIInput" % (state["LocalReadVectorWidth"])) return # for example, if the original ds_read is b32... # 1. if LoopIters = 5 (b32 x 5 times), WLR-Multiple = 2 (b64), then we can fit the WLR # 2. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 4 (b128), this is not allowed # 3. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 2 (b64), this is allowed if state["LoopIters"] % wlrMultiple != 0: - reject(state, "LocalReadVectorWidth %u cannot be distributed evenly, LoopIters %u should be divisible by WLR-Multiple %u" \ + reject(state, printRejectionReason, "LocalReadVectorWidth %u cannot be distributed evenly, LoopIters %u should be divisible by WLR-Multiple %u" \ % (state["LocalReadVectorWidth"], state["LoopIters"], wlrMultiple)) if state["LoopIters"] - (state["PrefetchLocalRead"] * wlrMultiple) < 0 : - reject(state, "with PrefetchLocalRead %u LoopIters %u LocalReadVectorWidth %u, not enough LoopIters to prefetch %ux%u iterations, " \ + reject(state, printRejectionReason, "with PrefetchLocalRead %u LoopIters %u LocalReadVectorWidth %u, not enough LoopIters to prefetch %ux%u iterations, " \ % (state["PrefetchLocalRead"],state["LoopIters"],state["LocalReadVectorWidth"], state["PrefetchLocalRead"] , wlrMultiple) ) # # reject conditions with lower performance # if state["ScheduleIterAlg"] == 2 and \ # (state["ExpandPointerSwap"] != 1 or state["LoopIters"] != 1 or state["ScheduleGlobalRead"] != 1): - # reject(state, "ScheduleIterAlg 2 only work with EPS1_SGR1, LoopIter=1") + # reject(state, printRejectionReason, "ScheduleIterAlg 2 only work with EPS1_SGR1, LoopIter=1") if state["TransposeLDS"] == 1: if not state["EnableMatrixInstruction"]: - reject(state, "TransposeLds Supports only in MatrixInstruction=1") + reject(state, printRejectionReason, "TransposeLds Supports only in MatrixInstruction=1") if state["ProblemType"]["TLUA"] and state["ProblemType"]["TLUB"]: # TODO: Now in rocBLAS, lot of logic yamls are Type=NT and TLDS=1? Why aren't they rejected and how to get rid of them? - reject(state, "TransposeLds requires TLUA=0 or TLUB=0") + reject(state, printRejectionReason, "TransposeLds requires TLUA=0 or TLUB=0") if state["EnableMatrixInstruction"]: # enable widerLocalRead if state["LocalReadVectorWidth"] > state["MIInputPerThread"]: @@ -3996,10 +3998,10 @@ def calcEpilogueTurns(factorDims: List) -> int: if not (state["PrefetchLocalRead"] >= state["LoopIters"] and state["InnerUnroll"] == 1) and \ not state["ClusterLocalRead"] and \ not state["InnerUnroll"] >= state["LocalReadVectorWidth"] // state["MIInputPerThread"]: - reject(state, "wider localRead only support ClusterLocalRead or (InnerUnroll > WiderLocalReadxN)") + reject(state, printRejectionReason, "wider localRead only support ClusterLocalRead or (InnerUnroll > WiderLocalReadxN)") if state["GlobalReadPerMfma"] > 1 and state["PrefetchGlobalRead"] == 2: - reject(state, "GlobalReadPerMfma need to be 1 if PGR2") + reject(state, printRejectionReason, "GlobalReadPerMfma need to be 1 if PGR2") if state["UseInstOffsetForGRO"] == -1: state["UseInstOffsetForGRO"] = 1 if state["DirectToLds"] else 0 @@ -4013,7 +4015,7 @@ def calcEpilogueTurns(factorDims: List) -> int: numVgprG2LB = roundUp((state["NumLoadsCoalescedB"] * state["NumLoadsPerpendicularB"] * \ state["GlobalReadVectorWidthB"] * bpeAB) / (float)(bpr)) if numVgprG2LA % 2 == 1 or numVgprG2LB % 2 == 1: - reject(state, "G2LA/B vgpr has bubble inside. Cannot use UnrollLoopSwapGlobalReadOrder=1.") + reject(state, printRejectionReason, "G2LA/B vgpr has bubble inside. Cannot use UnrollLoopSwapGlobalReadOrder=1.") if state["GlobalReadVectorWidthA"] != state["GlobalReadVectorWidthB"]: # TODO: Add a configuration to schedule better. state["ULSGRODoubleG2L"] = 1 @@ -4022,11 +4024,11 @@ def calcEpilogueTurns(factorDims: List) -> int: # G2LA/B vgpr index will jump. state["ULSGRODoubleG2L"] = 1 if state["ExpandPointerSwap"] == 1: - reject(state, "ExpandPointerSwap need to be 0 if UnrollLoopSwapGlobalReadOrder") + reject(state, printRejectionReason, "ExpandPointerSwap need to be 0 if UnrollLoopSwapGlobalReadOrder") if state["PrefetchGlobalRead"] != 2: - reject(state, "PrefetchGlobalRead need to be 2 if UnrollLoopSwapGlobalReadOrder") + reject(state, printRejectionReason, "PrefetchGlobalRead need to be 2 if UnrollLoopSwapGlobalReadOrder") if state["ProblemType"]["DataTypeA"].numBytes() != state["ProblemType"]["DataTypeB"].numBytes(): - reject(state, "UnrollLoopSwapGlobalReadOrder doesn't support mixed precision.") + reject(state, printRejectionReason, "UnrollLoopSwapGlobalReadOrder doesn't support mixed precision.") # guard against out of bounds reads # None: don't guard against ou @@ -4055,7 +4057,7 @@ def calcEpilogueTurns(factorDims: List) -> int: if bufferLoad and state["_UseSgprForGRO"] and state["EdgeType"]=="ShiftPtr": if not state["GuaranteeNoPartialA"] or not state["GuaranteeNoPartialB"] or not state["GuaranteeNoPartialMetadata"]: state["_UseSgprForGRO"] = False - #reject(state, "PBC with wide load has insufficient overlap guarantees- try GRVW=1 or adding appropriate Assert*ElementMultiple") + #reject(state, printRejectionReason, "PBC with wide load has insufficient overlap guarantees- try GRVW=1 or adding appropriate Assert*ElementMultiple") @@ -4064,7 +4066,7 @@ def calcEpilogueTurns(factorDims: List) -> int: cont1 = not state["GuaranteeNoPartialB"] cont2 = ((state["MatrixInstN"] % state["GlobalReadVectorWidthB"]) != 0) if cont1 and cont2: - reject(state, "MatrixInstN %u %% GlobalReadVectorWidthB %u must be 0" % \ + reject(state, printRejectionReason, "MatrixInstN %u %% GlobalReadVectorWidthB %u must be 0" % \ (state["MatrixInstN"], state["GlobalReadVectorWidthB"])) # Use SGPR to store an offset from GlobalReadOffsetA+0. @@ -4073,7 +4075,7 @@ def calcEpilogueTurns(factorDims: List) -> int: # individual vector registers doing bounds compares. if state["_UseSgprForGRO"] == 1 and (state["ProblemType"]["SwizzleTensorA"] or state["ProblemType"]["SwizzleTensorB"]): - reject(state, "UseSgprForGRO for Swizzle is not supported") + reject(state, printRejectionReason, "UseSgprForGRO for Swizzle is not supported") if state["_UseSgprForGRO"] == -1: # Don't use SGPR if it looks like we might not have enough - better to leave PBC enabled even if we have to use VGPR @@ -4090,45 +4092,45 @@ def calcEpilogueTurns(factorDims: List) -> int: state["_UseSgprForGRO"] = 1 if packedC0 and not state["GuaranteeNoPartialA"]: - reject(state, "packedC0 requires GuaranteeNoPartialA") + reject(state, printRejectionReason, "packedC0 requires GuaranteeNoPartialA") if packedC1 and not state["GuaranteeNoPartialB"]: - reject(state, "packedC1 requires GuaranteeNoPartialB") + reject(state, printRejectionReason, "packedC1 requires GuaranteeNoPartialB") if packedC0 or packedC1: state["_UseSgprForGRO"] = 0 if state["EdgeType"] != "ShiftPtr": - reject(state, "Packed dims requires EdgeType==ShiftPtr") + reject(state, printRejectionReason, "Packed dims requires EdgeType==ShiftPtr") if state["KernelLanguage"] == "Assembly": if not bufferLoad: - reject(state, "Packed dims for Assembly requires BufferLoad") + reject(state, printRejectionReason, "Packed dims for Assembly requires BufferLoad") if packedC0: # VectorWidth must not span tensor dim if state["KernelLanguage"] == "Source": if state["AssertFree0ElementMultiple"]=VectorWidth (for loads and stores)") + reject(state, printRejectionReason, "packedC0 Source requires AF0EM>=VectorWidth (for loads and stores)") else: if state["AssertFree0ElementMultiple"]=VectorWidth or not VectorStore (for stores)") + reject(state, printRejectionReason, "packedC0 Assembly requires AF0EM>=VectorWidth or not VectorStore (for stores)") state["AssignedDerivedParameters"] = True # Set E if state["ProblemType"]["UseE"]: if (state["_GlobalAccumulation"] == 'SingleBuffer') and state["GlobalSplitU"] > 1: - reject(state, "GlobalSplitU > 1 only compatible with MultipleBuffer") + reject(state, printRejectionReason, "GlobalSplitU > 1 only compatible with MultipleBuffer") if len(state["PackedC1IndicesX"]) > 1: - reject(state, "Use E does not support len(PackedC1IndicesX) > 1.") + reject(state, printRejectionReason, "Use E does not support len(PackedC1IndicesX) > 1.") if not state["BufferStore"]: - reject(state, "Use E only supports BufferStore due to no suppress no store.") + reject(state, printRejectionReason, "Use E only supports BufferStore due to no suppress no store.") if state["StoreRemapVectorWidth"] and (state["GlobalSplitU"] == 1): - reject(state, "Use E does not support StoreRemapVectorWidth if GSU == 1.") + reject(state, printRejectionReason, "Use E does not support StoreRemapVectorWidth if GSU == 1.") if state["GroupLoadStore"]: - reject(state, "Use E does not support GroupLoadStore.") + reject(state, printRejectionReason, "Use E does not support GroupLoadStore.") # Activation # Function call is set to false if GSU != 1 or Activation is not fused or ActivationType is not All. @@ -4137,24 +4139,24 @@ def calcEpilogueTurns(factorDims: List) -> int: state["ActivationFuncCall"] = False if state["ActivationAlt"]: - reject(state, "Currently does not accept ActivationAlt.") + reject(state, printRejectionReason, "Currently does not accept ActivationAlt.") # Bias reduction if state["ProblemType"]["UseBias"] and state["ProblemType"]["Gradient"]: if (state["_GlobalAccumulation"] == 'SingleBuffer') and state["GlobalSplitU"] > 1: - reject(state, "GlobalSplitU > 1 only compatible with MultipleBuffer for bias reduction") + reject(state, printRejectionReason, "GlobalSplitU > 1 only compatible with MultipleBuffer for bias reduction") if len(state["PackedC1IndicesX"]) > 1: - reject(state, "Bias reduction does not support len(PackedC1IndicesX) > 1.") + reject(state, printRejectionReason, "Bias reduction does not support len(PackedC1IndicesX) > 1.") if not state["BufferStore"]: - reject(state, "Bias reduction only supports BufferStore due to no suppress no store.") + reject(state, printRejectionReason, "Bias reduction only supports BufferStore due to no suppress no store.") if state["StoreRemapVectorWidth"] and (state["GlobalSplitU"] == 1): - reject(state, "Bias reduction does not support StoreRemapVectorWidth if GSU == 1.") + reject(state, printRejectionReason, "Bias reduction does not support StoreRemapVectorWidth if GSU == 1.") if state["GroupLoadStore"]: - reject(state, "Bias reduction does not support GroupLoadStore.") + reject(state, printRejectionReason, "Bias reduction does not support GroupLoadStore.") # Bias and ScaleAlphaVec if state["ProblemType"]["UseBias"] != 0 and state["ProblemType"]["UseScaleAlphaVec"] != 0 and state["ProblemType"]["UseBias"] != state["ProblemType"]["UseScaleAlphaVec"]: - reject(state, "When both UseBias and UseScaleAlphaVec are enabled then UseBias and UseScaleAlphaVec must have same settings.") + reject(state, printRejectionReason, "When both UseBias and UseScaleAlphaVec are enabled then UseBias and UseScaleAlphaVec must have same settings.") # ScaleAB or ScaleABVec if state["ProblemType"]["DataTypeA"] != state["ProblemType"]["DataTypeB"] and \ @@ -4170,21 +4172,21 @@ def calcEpilogueTurns(factorDims: List) -> int: # if state["GlobalSplitU"] > 1: # if state["ProblemType"]["SupportUserArgs"] and state["_GlobalAccumulation"] != 'MultipleBufferSingleKernel': - # reject(state, "Currently SupportUserArgs does not support GSU > 1.") - + # reject(state, printRejectionReason, "Currently SupportUserArgs does not support GSU > 1.") + if state["_GlobalAccumulation"] == 'MultipleBufferSingleKernel': - if (state["NumElementsPerBatchStore"] == 1): - reject(state, "too many store at MultipleBufferSingleKernel direct reject") + if state["NumElementsPerBatchStore"] == 1: + reject(state, printRejectionReason, "too many store at MultipleBufferSingleKernel direct reject") if state["ProblemType"]["UseScaleCD"]: - reject(state, "MultipleBufferSingleKernel not support UseScaleCD yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support UseScaleCD yet") if state["ProblemType"]["UseE"]: - reject(state, "MultipleBufferSingleKernel not support UseE yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support UseE yet") if state["ProblemType"]["BiasSrc"] != "D": - reject(state, "MultipleBufferSingleKernel not support BiasSrc not D yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support BiasSrc not D yet") if state["ProblemType"]["DataType"].isDouble(): - reject(state, "MultipleBufferSingleKernel not support " + str(state["ProblemType"]["DataType"]) + " yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support " + str(state["ProblemType"]["DataType"]) + " yet") if state["ProblemType"]["Sparse"] != 0: - reject(state, "MultipleBufferSingleKernel not support sparse yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support sparse yet") #Need to force disabling PreloadKernArgs if compiler does not support #Can not just reject the solution since the user library may find any solutions @@ -4293,6 +4295,7 @@ def getNameFull(state, splitGSU: bool): # Get Name Min @ staticmethod def getNameMin(state, requiredParameters, splitGSU: bool, ignoreInternalArgs = False): + if isCustomKernelConfig(state): return state["CustomKernelName"] diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index b0dee9589d..103831e4a1 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -33,9 +33,11 @@ from datetime import datetime from pathlib import Path +from typing import List from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ - assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR + assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ + detectGlobalCurrentISA, IsaVersion from Tensile.Toolchain.Assembly import AssemblyToolchain from Tensile.Toolchain.Source import SourceToolchain from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults @@ -62,7 +64,9 @@ def executeStepsInConfig( asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - debugConfig: DebugConfig + debugConfig: DebugConfig, + currentIsa: IsaVersion, + deviceId: int ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -88,7 +92,7 @@ def executeStepsInConfig( ############################################################################## if "BenchmarkProblems" in config: BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, \ - cCompiler, outputPath, buildTmpPath, config["ShortNames"], debugConfig) + cCompiler, outputPath, buildTmpPath, config["ShortNames"], debugConfig, currentIsa, deviceId) print1("") ############################################################################## @@ -105,7 +109,7 @@ def executeStepsInConfig( libraryLogicConfig = config["LibraryLogic"] else: libraryLogicConfig = {} - LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath, debugConfig) + LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason) print1("") else: print1("# LibraryLogic already done.") @@ -138,7 +142,7 @@ def splitExtraParameters(par): value = eval(value) return (key, value) - argParser.add_argument("-d", "--device", dest="device", type=int, \ + argParser.add_argument("-d", "--device", dest="device", default=0, type=int, \ help="override which device to benchmark") argParser.add_argument("-p", "--platform", dest="platform", type=int, \ help="override which OpenCL platform to benchmark") @@ -176,9 +180,6 @@ def argUpdatedGlobalParameters(args): """ rv = {} # override config with command-line options - if args.device: - print1("# Command-line override: Device") - rv["Device"] = args.device if args.platform: print1("# Command-line override: Platform") rv["Platform"] = args.platform @@ -422,7 +423,7 @@ def Tensile(userArgs): config["UseCache"] = useCache globalParameters["ConfigPath"] = configPaths - device_id = config["GlobalParameters"].get("Device", globalParameters["Device"]) + device_id = config["GlobalParameters"].get("Device", int(args.device)) UseEffLike = config["GlobalParameters"].get("UseEffLike", globalParameters["UseEffLike"]) if 'LibraryLogic' in config and UseEffLike: @@ -458,7 +459,9 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler, debugConfig) + currentIsa = detectGlobalCurrentISA(device_id) + + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler, debugConfig, currentIsa, device_id) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 3cdc82ba30..76fd524913 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -33,8 +33,10 @@ from Tensile import SOURCE_PATH, LibraryIO from Tensile.Common import ( - HR, CHeader, + DebugConfig, + detectGlobalCurrentISA, + HR, IsaVersion, ParallelMap2, SemanticVersion, @@ -45,8 +47,10 @@ isaToGfx, print1, print2, + printWarning, printExit, state, + SUPPORTED_ISA, tqdm, ) from Tensile.CustomYamlLoader import load_logic_gfx_arch @@ -198,10 +202,10 @@ def writeSolutionsAndKernels( kernelHelperObjs, kernelWriterAssembly, splitGSU: bool, + cmdlineArchs: List[str], errorTolerant=False, generateSourcesAndExit=False, compress=True, - fromTensile=False, useShortNames=False, ): codeObjectFiles = [] @@ -263,7 +267,7 @@ def assemble(ret): asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames ) buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, cmdlineArchs ) return codeObjectFiles, numKernels @@ -276,8 +280,8 @@ def writeSolutionsAndKernelsTCL( kernels, kernelHelperObjs, kernelWriterAssembly, + cmdlineArchs: List[str], compress=True, - fromTensile=False, useShortNames=False, ): @@ -329,7 +333,7 @@ def assemble(ret): writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, cmdlineArchs ) return len(uniqueAsmKernels) @@ -337,13 +341,13 @@ def assemble(ret): @timing def getSolutionAndKernelWriters( - solutions, kernels, assembler: str, assemblerVersion: SemanticVersion + solutions, kernels, assembler: str, assemblerVersion: SemanticVersion, currentIsa: IsaVersion ): kernelSerialNaming = Solution.getSerialNaming(kernels) solutionMinNaming = Solution.getMinNaming(solutions) kernelMinNaming = Solution.getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion + kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion, DebugConfig(), currentIsa ) return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) @@ -373,11 +377,11 @@ def generateKernelObjectsFromSolutions(solutions): kernelHelperObjs = [] kernelNames = set() kernelHelperNames = set() - + splitGSU = False for solution in solutions: solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = Solution.getKeyNoInternalArgs(kernel, splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -408,7 +412,9 @@ def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): masterLibraries = {} nextSolIndex = 0 - fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(archs)) + splitGSU = False # TODO make this configurable + printSolutionRejectionReason = False # TODO make this configurable + fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(splitGSU), itertools.repeat(printSolutionRejectionReason), itertools.repeat(archs)) def libraryIter(lib: MasterSolutionLibrary): if len(lib.solutions): @@ -575,11 +581,11 @@ def validLogicFile(p: Path): print2(f"# LibraryLogicFiles: {len(logicFiles)}") for logicFile in logicFiles: print2("# %s" % logicFile) - + currentIsa = detectGlobalCurrentISA(0) solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion + solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion, currentIsa ) copyStaticFiles(outputPath) @@ -591,28 +597,29 @@ def validLogicFile(p: Path): kernels, kernelHelperObjs, kernelWriterAssembly, - arguments["ShortNames"], + archs, + useShortNames=arguments["ShortNames"], compress=arguments["UseCompression"], ) - archs = [ + archs = [ # is this really different than the other archs above? isaToGfx(arch) - for arch in globalParameters["SupportedISA"] + for arch in SUPPORTED_ISA if globalParameters["AsmCaps"][arch]["SupportedISA"] ] newLibraryDir = ensurePath(os.path.join(outputPath, "library")) - + splitGSU = False for archName, newMasterLibrary in masterLibraries.items(): if archName in archs: if globalParameters["LazyLibraryLoading"]: masterFile = os.path.join(newLibraryDir, "TensileLibrary_lazy_" + archName) else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_" + archName) - newMasterLibrary.applyNaming(kernelMinNaming) + newMasterLibrary.applyNaming(splitGSU, kernelMinNaming) LibraryIO.write(masterFile, state(newMasterLibrary), arguments["LibraryFormat"]) for name, lib in newMasterLibrary.lazyLibraries.items(): filename = os.path.join(newLibraryDir, name) - lib.applyNaming(kernelMinNaming) + lib.applyNaming(splitGSU, kernelMinNaming) LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) if not globalParameters["KeepBuildTmp"]: diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index 21511588fe..493e4b9749 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -27,9 +27,10 @@ from dataclasses import dataclass from typing import Tuple -from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs, IsaVersion, print1 from .Formatting import __TI_DEBUG_LEVEL__, printExit +from timeit import default_timer as timer def fastdeepcopy(x): # Note: Some object can't be pickled @@ -58,26 +59,31 @@ class IsaInfo: @dataclass class kernelInfo: - isa: Tuple[int, int, int] + isa: IsaVersion wavefrontSize: int = 64 - def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool=False) -> None: + def init(self, isaVersion: IsaVersion, assemblerPath: str, debug: bool=False) -> None: + if isinstance(isaVersion, int): + raise Exception(f"{isaVersion}") with self._lock: if len(self._kernelInfo) > 1000: self._kernelInfo = _removeIdent(self._kernelInfo) self._kernelInfo[threading.get_ident()] = TensileInstructions.kernelInfo(isa=isaVersion) if isaVersion not in self._isaInfo: # type: ignore + start = timer() asmCaps = initAsmCaps(isaVersion, assemblerPath, debug) archCaps = initArchCaps(isaVersion) regCaps = initRegisterCaps(isaVersion, archCaps) asmBugs = initAsmBugs(asmCaps) self._isaInfo[isaVersion] = TensileInstructions.IsaInfo(assemblerPath, # type: ignore asmCaps, archCaps, regCaps, asmBugs) + stop = timer() + print1(f"time: {stop-start}") def setDebugLevel(self, level: int) -> None: __TI_DEBUG_LEVEL__ = level - def setKernelInfo(self, isaVersion: Tuple[int, int, int], wavefrontSize: int) -> None: + def setKernelInfo(self, isaVersion: IsaVersion, wavefrontSize: int) -> None: if isaVersion not in self._isaInfo: # type: ignore import traceback printExit(f"Current isa {str(isaVersion)} not initialized. Initialized isas are {str(self._isaInfo.keys())}, traceback: {traceback.format_stack()}") diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 59ee225071..ba786a96e7 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -32,7 +32,7 @@ from timeit import default_timer as timer from typing import List, Union -from ..Common import globalParameters, print1, print2, ensurePath, splitArchs +from ..Common import globalParameters, print1, print2, ensurePath class SourceToolchain: def __init__(self, compiler: str, bundler: str, buildIdKind: str, asanBuild: bool=False, saveTemps: bool=False): @@ -170,7 +170,7 @@ def buildSourceCodeObjectFiles( tmpObjDir: Union[Path, str], includeDir: Union[Path, str], kernelPath: Union[Path, str], - fromTensile: bool + cmdlineArchs: List[str] ) -> List[str]: """Compiles a HIP source code file into a code object file. @@ -197,8 +197,6 @@ def buildSourceCodeObjectFiles( coPathsRaw = [] coPaths= [] - _, cmdlineArchs = splitArchs(globalParameters, fromTensile) - objPath = str(tmpObjDir / objFilename) toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) From af9f765235363bd65ba800c2c01ab6c0f09bccae Mon Sep 17 00:00:00 2001 From: David Dixon Date: Fri, 14 Feb 2025 17:01:59 +0000 Subject: [PATCH 050/105] remove unused globalParrams reference --- tensilelite/Tensile/Components/GlobalWriteBatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index 3e1206a0a4..a979cb1ff5 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..Common import globalParameters, DataDirection, SemanticVersion +from ..Common import DataDirection, SemanticVersion from ..Component import GlobalWriteComponents from ..SolutionStructs import Solution from ..Activation import ActivationModule, ActivationType From a42f80063635d7236a9f49d854662096651b8f52 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Fri, 14 Feb 2025 20:07:13 +0000 Subject: [PATCH 051/105] Removes current isa usage --- tensilelite/Tensile/BenchmarkProblems.py | 7 ++- tensilelite/Tensile/ClientWriter.py | 4 +- tensilelite/Tensile/Common/Architectures.py | 2 +- .../Tensile/Common/GlobalParameters.py | 49 +++++++------------ tensilelite/Tensile/GenerateSummations.py | 4 +- tensilelite/Tensile/KernelWriter.py | 5 +- tensilelite/Tensile/LibraryLogic.py | 22 ++++----- tensilelite/Tensile/SolutionStructs.py | 31 +++++------- tensilelite/Tensile/Tensile.py | 15 +++--- .../Tensile/TensileCreateLibrary/Run.py | 20 +++++--- 10 files changed, 67 insertions(+), 92 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 0b6450bd8f..2af00e0b17 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -314,10 +314,9 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize "to see why each parameter combination was rejected." printExit(msg) - if globalParameters["PrintLevel"] >= 1: - for solution in solutions: - print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution, debugConfig.splitGSU))) - print2(HR) + for solution in solutions: + print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution, debugConfig.splitGSU))) + print2(HR) # write benchmarkFiles prevCount = len(solutions) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 6c50b4260e..65ba3c6476 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -35,7 +35,7 @@ from . import ClientExecutable from . import LibraryIO from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, \ - LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR + LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR, detectGlobalCurrentISA from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex @@ -513,7 +513,7 @@ def param(key, value): libraryFile = os.path.join(sourceDir, "library", libraryFilename) param("library-file", libraryFile) - currentGFXName = isaToGfx(globalParameters["CurrentISA"]) + currentGFXName = isaToGfx(detectGlobalCurrentISA(deviceId)) for coFile in codeObjectFiles: if 'gfx' not in coFile or currentGFXName in coFile: param("code-object", os.path.join(sourceDir,coFile)) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index c8043b27a1..c55afc70a5 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -158,7 +158,7 @@ def _detectGlobalCurrentISA(detectionTool, deviceId: int): arch = gfxToIsa(line.strip()) if arch is not None: if arch in SUPPORTED_ISA: - print("# Detected local GPU with ISA: " + isaToGfx(arch)) + print(f"# Detected GPU {deviceId} with ISA: " + isaToGfx(arch)) archList.append(arch) if len(archList) > 0: result = archList[deviceId] diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 893e1eed84..8165a7668b 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -29,13 +29,15 @@ import time from collections import OrderedDict from copy import deepcopy +from typing import List from Tensile import __version__ -from .Architectures import isaToGfx, SUPPORTED_ISA, detectGlobalCurrentISA +from .Architectures import isaToGfx, SUPPORTED_ISA from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps from .Types import IsaVersion -from .Utilities import locateExe, versionIsCompatible, print1, print2, printExit, printWarning +from .Utilities import locateExe, versionIsCompatible, print1, print2, printExit, printWarning, \ + verbosity startTime = time.time() @@ -46,9 +48,6 @@ globalParameters["PerformanceMetric"] = ( "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} ) -globalParameters["PrintLevel"] = ( - 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose -) globalParameters["ClientLogLevel"] = ( 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) ) @@ -252,7 +251,6 @@ ) # internal, i.e., gets set during startup -globalParameters["CurrentISA"] = (0, 0, 0) globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi @@ -260,7 +258,7 @@ # default runtime is selected based on operating system, user can override if os.name == "nt": - globalParameters["RuntimeLanguage"] = "HIP" # "OCL" + globalParameters["RuntimeLanguage"] = "HIP" else: globalParameters["RuntimeLanguage"] = "HIP" @@ -1466,16 +1464,15 @@ def printTable(rows): print() -def printCapTable(parameters): +def printCapTable(parameters, targetIsas: List[IsaVersion]): import itertools - archs = [(0, 0, 0)] + SUPPORTED_ISA - gfxNames = list(map(isaToGfx, archs)) + gfxNames = list(map(isaToGfx, targetIsas)) headerRow = ["cap"] + gfxNames def capRow(caps, cap): - return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] + return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in targetIsas] allAsmCaps = set( itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) @@ -1492,7 +1489,7 @@ def capRow(caps, cap): printTable([headerRow] + asmCapRows + archCapRows) -def assignGlobalParameters(config, cxxCompiler=None): +def assignGlobalParameters(config, targetIsas: List[IsaVersion], cxxCompiler=None): """ Assign Global Parameters Each global parameter has a default parameter, and the user @@ -1571,40 +1568,30 @@ def assignGlobalParameters(config, cxxCompiler=None): if "CodeObjectVersion" in config: globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - # read current gfx version - currentIsa = detectGlobalCurrentISA(0) - globalParameters["CurrentISA"] = currentIsa - if globalParameters["CurrentISA"] == (0, 0, 0): - printWarning( - "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." - % SUPPORTED_ISA - ) - globalParameters["AsmCaps"] = {} globalParameters["ArchCaps"] = {} globalParameters["AsmBugs"] = {} - # We shouldn't need to do this for all ISAs... - # Why not only do this for ISAs that we are building. - for v in SUPPORTED_ISA + [IsaVersion(0, 0, 0)]: - + # do we really need to ad [IsaVersion(0,0,0)] to targetIsas? + for v in targetIsas: globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) globalParameters["ArchCaps"][v] = initArchCaps(v) globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) + if verbosity >= 1: + printCapTable(globalParameters, targetIsas) - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) - - SUPPORTED_ISA = list( + # This seems like we are restating line 1577 + # should we error out here if an isa was requested that we don't support? + isaList = list( [ i - for i in SUPPORTED_ISA + for i in targetIsas if globalParameters["AsmCaps"][i]["SupportedISA"] ] ) - validParameters["ISA"] = [(0, 0, 0), *SUPPORTED_ISA] + validParameters["ISA"] = [IsaVersion(0, 0, 0), *isaList] # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index 3b791d1c92..dfae4abd6c 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -37,7 +37,7 @@ from . import ClientWriter from .Common import assignGlobalParameters, ensurePath, globalParameters, \ - printExit, isaToGfx, gfxToSwCodename + printExit, isaToGfx, gfxToSwCodename, detectGlobalCurrentISA from .SolutionStructs import ProblemSizes from .Toolchain.Validators import ToolchainDefaults, validateToolchain @@ -67,7 +67,7 @@ def GenerateSummations(userArgs): assignGlobalParameters({}) cxxCompiler, cCompiler = validateToolchain(ToolchainDefaults.CXX_COMPILER, ToolchainDefaults.C_COMPILER) - currentISA = globalParameters["CurrentISA"] + currentISA = detectGlobalCurrentISA(0) gfxName = isaToGfx(currentISA) commonName = gfxToSwCodename(gfxName) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 9382c658dc..55cd1ceca3 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -5002,11 +5002,8 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): self.states.kernel = kernel self.states.language = "ASM" # we already do this in the solution ctor - #self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else self.currentIsa - if not globalParameters["AsmCaps"][self.states.version]["SupportedISA"]: - self.states.version = (9,0,0) - printWarning(f"ISA: {self.version} is not supported; overriding with {self.states.version}") + assert globalParameters["AsmCaps"][self.states.version]["SupportedISA"] return code diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index a51642a147..422752fc03 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -25,7 +25,7 @@ from pathlib import Path from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -107,7 +107,7 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL printExit("Bad KeepLogic=%u"%globalParameters["KeepLogic"]) # print raw data - if globalParameters["PrintLevel"] >= 2: + if verbosity >= 2: line = "After Removals:\n" numOther = 1 for size in logicAnalyzer.numProblemSizes: @@ -689,7 +689,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): currentIndexRange[self.indexOrder[2]][0], \ currentIndexRange[self.indexOrder[3]][0]) tab = self.tab[cii] - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("\n%s"%tab) currentIndex = self.indexOrder[currentIndexIndex] print2("%senRule(%s)" % (tab, currentIndexRange)) @@ -715,7 +715,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): print2("%sSingleProblem & LastIndex :: winnerIdx<0; returning" % (tab) ) return None ruleList.append([-1, winnerIdx]) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("%") ######################################## @@ -731,7 +731,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): return None rule = [ -1, nextRule ] ruleList.append(rule) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("%") else: @@ -784,7 +784,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): initialRule = [ currentIndexRange[currentIndex][0], nextRule ] ruleList.append(initialRule) print2("%sMultiProblem::InitialRuleList=%s" % (tab, ruleList)) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("#") ######################################## @@ -809,7 +809,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if winnerIdx < 0: ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE print2("%sUpdating range b/c None" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue else: @@ -822,7 +822,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if nextRule == None: ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE print2("%sUpdating b/c None" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue else: @@ -833,7 +833,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if candidateRule[1] == priorRule[1]: print2("%sCandidateRule==PriorRule; just updating prior" % (tab)) ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue @@ -873,14 +873,14 @@ def enRule(self, currentIndexIndex, currentIndexRange): if True: # or candidateRuleScore < priorRuleScore: ruleList.append(candidateRule) print2("%sAppending b/c Different" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("#") ######################################## # prior wins else: print2("%sPrior Rule Wins" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(".") ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index faeb9c9dbe..aed683d99e 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1061,7 +1061,7 @@ def isExtractableIndex(ks, index, tc='x'): class Solution(collections.abc.Mapping): ######################################## # need to be sure PSRR is passing to all fxns - def __init__(self, config, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: List[IsaVersion], cxxCompiler: str, srcName: str = ""): + def __init__(self, config, splitGSU: bool, printSolutionRejectionReason: bool, targetIsas: List[IsaVersion], cxxCompiler: str, srcName: str = ""): self._name = None self.cxxCompiler = cxxCompiler self.srcName = srcName @@ -1087,18 +1087,12 @@ def __init__(self, config, splitGSU: bool, printSolutionRejectionReason: bool, s assignParameterWithDefault(self._state, key, config, defaultSolution) if 'ISA' not in self._state: if 'ISA' in config: - if not globalParameters["AsmCaps"][tuple(config['ISA'])]["SupportedISA"]: - defaultIsa = [9,0,0] - print("warning: ISA:", config['ISA'], " is not supported; overriding with ", defaultIsa) - self._state['ISA'] = defaultIsa - else: - self._state['ISA'] = config['ISA'] + assert globalParameters["AsmCaps"][tuple(config['ISA'])]["SupportedISA"] + isa = config['ISA'] + self._state['ISA'] = IsaVersion(isa[0], isa[1], isa[2]) else: - # Assembly by default - self._state['ISA'] = list(globalParameters["CurrentISA"]) - if 'KernelLanguage' in config: - if config['KernelLanguage'] != 'Assembly': - self._state['ISA'] = [0,0,0] + printWarning(f"ISA not set on config using {targetIsas[0]}.") + self._state['ISA'] = targetIsas[0] if "CodeObjectVersion" not in self._state: if "CodeObjectVersion" in config: @@ -1118,7 +1112,7 @@ def __init__(self, config, splitGSU: bool, printSolutionRejectionReason: bool, s Solution.assignDerivedParameters(self._state, splitGSU, printSolutionRejectionReason) self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None - self.initHelperKernelObjects(supportedISA) + self.initHelperKernelObjects(targetIsas) # these keys are copied from ProblemType to internal that may be overridden InternalKeys = ["UseSgprForGRO","VectorStore"] @@ -1606,10 +1600,7 @@ def setGlobalLoadTileDimFractional(state, tc, depthU): # and a possible opportunity to handle the lsc grvw = state["GlobalReadVectorWidth%s"%tc] minGrvw = 2 if state["ProblemType"]["DataType"].isHalf() and \ - globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 - # TODO- check this for int8 and fractional load - # minGrvw = 4 if state["ProblemType"]["DataType"].isInt8() and \ - # globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 + globalParameters["ArchCaps"][state["ISA"]]["HasEccHalf"] else 1 # why were we using current ISA here? bestVw = -1 while grvw >= minGrvw: # Per instruction across the entire group: @@ -1719,7 +1710,7 @@ def setGlobalLoadTileDimFractional(state, tc, depthU): @staticmethod def MatrixInstructionToMIParameters(state, printRejectionReason: bool): - isa = tuple(state["ISA"]) + isa = state["ISA"] if len(state["MatrixInstruction"]) == 9: mi = state["MatrixInstruction"] state["MatrixInstruction"] = [state["MatrixInstruction"][0],state["MatrixInstruction"][1],state["MatrixInstruction"][2],state["MatrixInstruction"][3]] @@ -2531,7 +2522,7 @@ def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool): if state["KernelLanguage"] == "Assembly" \ and state["ProblemType"]["DataType"].isHalf(): - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if globalParameters["ArchCaps"][state["ISA"]]["HasEccHalf"]: if not state["ProblemType"]["HighPrecisionAccumulate"] and state["AssertFree0ElementMultiple"] % 2 != 0: # beta-on-edge has AF0EM requirement except for HPA kernels reject(state, printRejectionReason, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") @@ -3041,7 +3032,7 @@ def calSwizzleK(state, tc): validDepthU = False if validDepthU and state["KernelLanguage"] == "Assembly": - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if globalParameters["ArchCaps"][state["ISA"]]["HasEccHalf"]: if state["ProblemType"]["DataType"].numRegisters() == 0.5 and (not state["ProblemType"]["HighPrecisionAccumulate"]): if state["GlobalReadVectorWidthA"] == 1 or state["GlobalReadVectorWidthB"] == 1: reject(state, printRejectionReason, "HalfEcc requires HPA if glvw = 1") diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 0fac822319..10a3ee1eea 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -37,7 +37,7 @@ from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ - detectGlobalCurrentISA, IsaVersion + detectGlobalCurrentISA, IsaVersion, verbosity from Tensile.Toolchain.Assembly import AssemblyToolchain from Tensile.Toolchain.Source import SourceToolchain from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults @@ -189,12 +189,8 @@ def argUpdatedGlobalParameters(args): if args.CodeObjectVersion: print1("# Command-line override: CodeObjectVersion") rv["CodeObjectVersion"] = args.CodeObjectVersion - if args.verbose: - print1("# Command-line override: PrintLevel") - rv["PrintLevel"] = 2 if args.debug: print1("# Command-line override: Debug") - rv["PrintLevel"] = 2 rv["CMakeBuildType"] = "Debug" if args.client_lock: rv["ClientExecutionLockPath"] = args.client_lock @@ -362,13 +358,15 @@ def Tensile(userArgs): addCommonArguments(argParser) args = argParser.parse_args(userArgs) - configPaths = args.ConfigFile altFormat = args.AlternateFormat useCache = args.useCache outputPath = Path(ensurePath(os.path.abspath(args.OutputPath))) print1(f"# OutputPath: {str(outputPath)}") + global verbosity + verbosity = 2 if (args.debug or args.verbose) else 1 + if altFormat and len(configPaths) > 2: printExit("Only 1 or 2 config_files are accepted for the alternate config format: " "the alternate config file and an optional size list") @@ -440,7 +438,8 @@ def Tensile(userArgs): store_max_frequency(max_frequency) cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) - assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) + currentIsa = detectGlobalCurrentISA(device_id) + assignGlobalParameters(config.get("GlobalParameters", {}), [currentIsa], cxxCompiler) asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) @@ -459,8 +458,6 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - currentIsa = detectGlobalCurrentISA(device_id) - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler, debugConfig, currentIsa, device_id) def TensileConfigPath(*args): diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 47083549ef..06fc0ecdd8 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -36,6 +36,7 @@ CHeader, DebugConfig, detectGlobalCurrentISA, + gfxToIsa, HR, IsaVersion, ParallelMap2, @@ -241,7 +242,7 @@ def writeSolutionsAndKernels( asmIter = zip( itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), itertools.repeat(useShortNames), asmKernels ) - asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels", return_as="generator_unordered") + asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels", return_as="list") removeInvalidSolutionsAndKernels( asmResults, asmKernels, solutions, errorTolerant, verbosity, splitGSU ) @@ -326,7 +327,7 @@ def assemble(ret): uniqueAsmKernels, "Generating assembly kernels", multiArg=False, - return_as="generator_unordered" + return_as="list" ) buildAssemblyCodeObjectFiles( asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames @@ -504,6 +505,8 @@ def run(): print2("") arguments = parseArguments() + global verbosity + verbosity = arguments["PrintLevel"] outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( arguments["CxxCompiler"], @@ -521,7 +524,12 @@ def run(): print1(f"# Architecture(s): {arguments['Architecture']}") print1(f"# Library Format: {arguments['LibraryFormat']}") - assignGlobalParameters(arguments, cxxCompiler) + if ";" in arguments["Architecture"]: + archs = arguments["Architecture"].split(";") + else: + archs = arguments["Architecture"].split("_") + targetIsas = [gfxToIsa(a) for a in archs] + assignGlobalParameters(arguments, targetIsas, cxxCompiler) asmToolchain = AssemblyToolchain( assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"] @@ -537,10 +545,6 @@ def run(): if not os.path.exists(arguments["LogicPath"]): printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") - if ";" in arguments["Architecture"]: - archs = arguments["Architecture"].split(";") - else: - archs = arguments["Architecture"].split("_") logicArchs = set() for arch in archs: if arch in architectureMap: @@ -606,7 +610,7 @@ def validLogicFile(p: Path): archs = [ # is this really different than the other archs above? isaToGfx(arch) - for arch in SUPPORTED_ISA + for arch in targetIsas if globalParameters["AsmCaps"][arch]["SupportedISA"] ] newLibraryDir = ensurePath(os.path.join(outputPath, "library")) From 4ef1bb976aa14b5baa5b411ada3042f87d181446 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sat, 15 Feb 2025 15:51:16 +0000 Subject: [PATCH 052/105] Serialization bug --- tensilelite/Tensile/Contractions.py | 2 -- tensilelite/Tensile/KernelWriter.py | 2 +- tensilelite/Tensile/LibraryIO.py | 9 +++--- tensilelite/Tensile/SolutionStructs.py | 4 +-- tensilelite/Tensile/Tensile.py | 45 ++++++++++++++++++-------- tensilelite/Tensile/Tests/conftest.py | 4 +-- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 83813d3267..ffc4e844e3 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -700,8 +700,6 @@ def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool if 'ISA' not in d: if d['KernelLanguage'] == 'Assembly': d['ISA'] = gfxToIsa(deviceInfo[1]) - else: - d['ISA'] = [0,0,0] if 'CUCount' not in d: d['CUCount'] = None diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 55cd1ceca3..b15af8ca9b 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -4313,7 +4313,7 @@ def readWriteVectors(mat, vw, kernel): # Special dependency cases if kernel["ProblemType"]["ComputeDataType"].isDouble(): if kernel["MatrixInstruction"] == [4, 4, 4, 4]: - if kernel['ISA'] == [9,0,10]: + if kernel['ISA'] == IsaVersion(9,0,10): self.states.miDependency = 4 diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index fe5395cdfb..e0f6898198 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -26,8 +26,7 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__, \ - IsaVersion +from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__ from typing import NamedTuple, List import os @@ -149,6 +148,8 @@ def writeSolutions(filename, problemSizes, biasTypeArgs, activationArgs, solutio if "DataTypeMetadata" in solutionState["ProblemType"]: solutionState["ProblemType"]["DataTypeMetadata"] = \ solutionState["ProblemType"]["DataTypeMetadata"].value + isa = solutionState["ISA"] + solutionState["ISA"] = [isa[0], isa[1], isa[2]] solutionStates.append(solutionState) # write dictionaries with open(filename, "w") as f: @@ -277,8 +278,6 @@ def parseLibraryLogicData(data, srcFile, cxxCompiler, splitGSU: bool, printSolut def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: if solutionState["KernelLanguage"] == "Assembly": solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) - else: - solutionState["ISA"] = IsaVersion(0, 0, 0) solutionState["CUCount"] = data["CUCount"] # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False @@ -468,6 +467,8 @@ def createLibraryLogic(schedulePrefix, architectureName, deviceNames, libraryTyp if "DataTypeMetadata" in solutionState["ProblemType"]: solutionState["ProblemType"]["DataTypeMetadata"] = \ solutionState["ProblemType"]["DataTypeMetadata"].value + isa = solutionState["ISA"] + solutionState["ISA"] = [isa[0], isa[1], isa[2]] solutionList.append(solutionState) if tileSelection: diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index aed683d99e..0574ee34cc 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1392,7 +1392,7 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool) state["tailLoopOptA"] = True state["tailLoopOptB"] = True - if (tuple(state["ISA"]) != (9, 4, 2)) or \ + if (state["ISA"] != IsaVersion(9, 4, 2)) or \ (state["ProblemType"]["Sparse"]): state["tailLoopOptA"] = False state["tailLoopOptB"] = False @@ -2283,7 +2283,7 @@ def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool): return if state["ProblemType"]["ComputeDataType"].isDouble(): # See [4,4,4,4] snop for more info - if state["MatrixInstruction"] == [4,4,4,4] and (not state['ISA'] == [9,0,10]) and state["ScheduleIterAlg"] == 3: + if state["MatrixInstruction"] == [4,4,4,4] and (not state['ISA'] == IsaVersion(9,0,10)) and state["ScheduleIterAlg"] == 3: reject(state, printRejectionReason, "Currently Matrix instructions [4,4,4,4] is disabled.") return else: diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 10a3ee1eea..f576b9100a 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -305,30 +305,49 @@ def store_max_frequency(max_frequency): def makeDebugConfig(config: dict) -> DebugConfig: - debugConfig = DebugConfig() + + enableAsserts = False + enableDebugA = False + enableDebugB = False + enableDebugC = False + expectedValueC = 16.0 + forceCExpectedValue = False + debugKernel = False + forceGenerateKernel = False + printSolutionRejectionReason = False + splitGSU = False if "EnableAsserts" in config: - debugConfig.enableAsserts = config["EnableAsserts"] + enableAsserts = config["EnableAsserts"] if "EnableDebugA" in config: - debugConfig.enableDebugA = config["EnableDebugA"] + enableDebugA = config["EnableDebugA"] if "EnableDebugB" in config: - debugConfig.enableDebugB = config["EnableDebugB"] + enableDebugB = config["EnableDebugB"] if "EnableDebugC" in config: - debugConfig.enableDebugC = config["EnableDebugC"] + enableDebugC = config["EnableDebugC"] if "ExpectedValueC" in config: - debugConfig.expectedValueC = config["ExpectedValueC"] + expectedValueC = config["ExpectedValueC"] if "ForceCExpectedValue" in config: - debugConfig.forceCExpectedValue = config["ForceCExpectedValue"] + forceCExpectedValue = config["ForceCExpectedValue"] if "DebugKernel" in config: - debugConfig.debugKernel = config["DebugKernel"] + debugKernel = config["DebugKernel"] if "ForceGenerateKernel" in config: - debugConfig.forceGenerateKernel = config["ForceGenerateKernel"] + forceGenerateKernel = config["ForceGenerateKernel"] if "PrintSolutionRejectionReason" in config: - debugConfig.printSolutionRejectionReason = config["PrintSolutionRejectionReason"] + printSolutionRejectionReason = config["PrintSolutionRejectionReason"] if "SplitGSU" in config: - debugConfig.splitGSU = config["SplitGSU"] + splitGSU = config["SplitGSU"] - return debugConfig + return DebugConfig(enableAsserts, + enableDebugA, + enableDebugB, + enableDebugC, + expectedValueC, + forceCExpectedValue, + debugKernel, + forceGenerateKernel, + printSolutionRejectionReason, + splitGSU) ################################################################################ @@ -449,7 +468,7 @@ def Tensile(userArgs): if "ShortNames" not in config: config["ShortNames"] = args.shortNames - debugConfig = makeDebugConfig(config) + debugConfig = makeDebugConfig(config["GlobalParameters"]) for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) diff --git a/tensilelite/Tensile/Tests/conftest.py b/tensilelite/Tensile/Tests/conftest.py index 894510feef..975ed7e6d6 100644 --- a/tensilelite/Tensile/Tests/conftest.py +++ b/tensilelite/Tensile/Tests/conftest.py @@ -139,8 +139,8 @@ def __enter__(self): Common.restoreDefaultGlobalParameters() if args.CxxCompiler: Common.globalParameters["CxxCompiler"] = args.CxxCompiler - - Common.assignGlobalParameters({}) + isa = Common.detectGlobalCurrentISA(args.device) + Common.assignGlobalParameters({}, isa) overrideParameters = Tensile.argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): From 208380b4813599dc115eec6c92b37e796de8f86e Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sat, 15 Feb 2025 17:41:24 +0000 Subject: [PATCH 053/105] Fix return from detect isa fxn --- tensilelite/Tensile/Common/Architectures.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index c55afc70a5..0897e8cdd9 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -160,11 +160,9 @@ def _detectGlobalCurrentISA(detectionTool, deviceId: int): if arch in SUPPORTED_ISA: print(f"# Detected GPU {deviceId} with ISA: " + isaToGfx(arch)) archList.append(arch) - if len(archList) > 0: - result = archList[deviceId] if process.returncode: print(f"{detectionTool} exited with code {process.returncode}") - return result if process.returncode == 0 else process.returncode + return archList[deviceId] if (len(archList) > 0 and process.returncode == 0) else process.returncode # locateExe silently fails which is not good From 35a235f39f93524d930de6bd1de2475d92f88644 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sat, 15 Feb 2025 22:09:32 +0000 Subject: [PATCH 054/105] Support cross compilation --- tensilelite/Tensile/KernelWriter.py | 6 +++--- tensilelite/Tensile/KernelWriterAssembly.py | 4 ++-- tensilelite/Tensile/TensileCreateLibrary/Run.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index b15af8ca9b..42f6cb3900 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -358,14 +358,13 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig, currentIsa: IsaVersion): + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler self.amdClangVersion = amdClangVersion # this is a bug self.ti = None self.debugConfig = debugConfig - self.currentIsa = currentIsa self.do = {} self.do["PreLoop"] = True @@ -5002,7 +5001,8 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): self.states.kernel = kernel self.states.language = "ASM" # we already do this in the solution ctor - self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else self.currentIsa + assert "ISA" in kernel + self.states.version = tuple(kernel["ISA"]) assert globalParameters["AsmCaps"][self.states.version]["SupportedISA"] return code diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 85598d06c1..d335470d7c 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -82,8 +82,8 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig, currentIsa: IsaVersion): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig, currentIsa) + def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion, debugConfig: DebugConfig): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig) def getSourceFileString(self, kernel, useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 06fc0ecdd8..c8acd0be50 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -344,13 +344,13 @@ def assemble(ret): @timing def getSolutionAndKernelWriters( - solutions, kernels, assembler: str, assemblerVersion: SemanticVersion, currentIsa: IsaVersion + solutions, kernels, assembler: str, assemblerVersion: SemanticVersion ): kernelSerialNaming = Solution.getSerialNaming(kernels) solutionMinNaming = Solution.getMinNaming(solutions) kernelMinNaming = Solution.getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion, DebugConfig(), currentIsa + kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion, DebugConfig() ) return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) @@ -587,11 +587,11 @@ def validLogicFile(p: Path): print2(f"# LibraryLogicFiles: {len(logicFiles)}") for logicFile in logicFiles: print2("# %s" % logicFile) - currentIsa = detectGlobalCurrentISA(0) + solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion, currentIsa + solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion ) copyStaticFiles(outputPath) From 10aee3b4d4a1018d687258cb44cccbbdec87416f Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sat, 15 Feb 2025 22:23:31 +0000 Subject: [PATCH 055/105] remove arg --- tensilelite/Tensile/BenchmarkProblems.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 2af00e0b17..f2cb47df22 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -148,7 +148,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelSerialNaming = Solution.getSerialNaming(kernels) kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, debugConfig, currentIsa) + kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion, debugConfig) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] From 6b579241b23f30b3d219258f9e04d2c2bdd06782 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sun, 16 Feb 2025 17:39:28 -0600 Subject: [PATCH 056/105] Fix 942 test failures --- tensilelite/Tensile/BenchmarkProblems.py | 2 +- tensilelite/Tensile/ClientWriter.py | 22 ++++++---------------- tensilelite/Tensile/Tensile.py | 5 +++-- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index f2cb47df22..1da701ef50 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -85,7 +85,7 @@ def _generateCustomKernelSolutions(problemType, customKernels, internalSupportPa solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, currentIsa, debugConfig) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, debugConfig, currentIsa) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 65ba3c6476..6283d6dc90 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -79,7 +79,7 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): +def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path, targetGfx: str, deviceId: int): libraryLogicPath = ensurePath(outputPath / LIBRARY_LOGIC_DIR) clientLibraryPath = ensurePath(outputPath / LIBRARY_CLIENT_DIR) @@ -97,7 +97,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): functions = [] functionNames = [] - createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler, config["ShortNames"]) + createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler, targetGfx, False) subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) coList = glob(os.path.join(clientLibraryPath, "library/*.co")) yamlList = glob(os.path.join(clientLibraryPath, "library/*.yaml")) @@ -105,7 +105,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): clientParametersPaths = [] for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary) \ - = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) + = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler, False, False, [targetGfx]) functions.append((scheduleName, problemType)) functionNames.append("tensile_%s" % (problemType)) problemSizes = ProblemSizesMock(exactLogic) if exactLogic else ProblemSizesMockDummy() @@ -150,6 +150,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): newLibrary=newLibrary, configBase="ClientParameters_%s"%str(ProblemType(problemType)), codeObjectFiles=coList, + deviceId=deviceId, tileAwareSelection=False, libraryFile=yamlList[0])) @@ -200,7 +201,7 @@ def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: return process.returncode -def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, useShortNames: bool=False): +def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, targetGfx, useShortNames: bool=False): import io runScriptFile = io.StringIO() @@ -218,7 +219,7 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, useSho if globalParameters["KeepBuildTmp"]: callCreateLibraryCmd += " --keep-build-tmp" - callCreateLibraryCmd += " --architecture=" + globalParameters["Architecture"] + callCreateLibraryCmd += " --architecture=" + targetGfx callCreateLibraryCmd += " --code-object-version=" + globalParameters["CodeObjectVersion"] callCreateLibraryCmd += " --cxx-compiler=" + cxxCompiler callCreateLibraryCmd += " --library-format=" + globalParameters["LibraryFormat"] @@ -231,17 +232,6 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, useSho return runScriptFile.getvalue() -def writeBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler): - filename = os.path.join(path, \ - "build.%s" % ("bat" if os.name == "nt" else "sh") ) - with open(filename, "w") as file: - file.write("#!/bin/bash\n\n") - file.write("set -ex\n") - file.write(getBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler)) - - if os.name != "nt": - os.chmod(filename, 0o777) - return filename def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, buildDir, configPaths=None): if configPaths is None: diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index f576b9100a..01346186e4 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -37,7 +37,7 @@ from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ - detectGlobalCurrentISA, IsaVersion, verbosity + detectGlobalCurrentISA, IsaVersion, verbosity, isaToGfx from Tensile.Toolchain.Assembly import AssemblyToolchain from Tensile.Toolchain.Source import SourceToolchain from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults @@ -123,7 +123,8 @@ def executeStepsInConfig( libraryClientConfig = config["LibraryClient"] else: libraryClientConfig = {} - ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath, config["ShortNames"]) + #libraryClientConfig["ShortNames"] = config["ShortNames"] + ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath, isaToGfx(currentIsa), deviceId) print1("") From 0d9b1269c3f1bef50e7fb9086cacdb171680c11e Mon Sep 17 00:00:00 2001 From: David Dixon Date: Sun, 16 Feb 2025 23:41:13 +0000 Subject: [PATCH 057/105] Updates to toolchain to accomodate rocm version --- tensilelite/Tensile/BenchmarkProblems.py | 70 ++-- tensilelite/Tensile/Common/Capabilities.py | 14 +- .../Tensile/Common/GlobalParameters.py | 23 +- tensilelite/Tensile/Contractions.py | 8 +- tensilelite/Tensile/KernelWriter.py | 34 +- tensilelite/Tensile/KernelWriterAssembly.py | 35 +- tensilelite/Tensile/KernelWriterConversion.py | 2 +- tensilelite/Tensile/LibraryIO.py | 50 ++- tensilelite/Tensile/SolutionLibrary.py | 21 +- tensilelite/Tensile/SolutionStructs.py | 16 +- tensilelite/Tensile/Tensile.py | 30 +- .../Tensile/TensileCreateLibrary/Run.py | 94 ++--- tensilelite/Tensile/Toolchain/Assembly.py | 135 ++----- tensilelite/Tensile/Toolchain/Component.py | 339 ++++++++++++++++++ tensilelite/Tensile/Toolchain/Source.py | 132 ++----- tensilelite/Tensile/Toolchain/Validators.py | 16 - 16 files changed, 607 insertions(+), 412 deletions(-) create mode 100644 tensilelite/Tensile/Toolchain/Component.py diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index f42f20e468..e7978e3a82 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -49,7 +49,7 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, \ - debugConfig: DebugConfig, currentIsa: IsaVersion, isaInfoMap: Dict[str, IsaInfo]): + debugConfig: DebugConfig, isaInfoMap: Dict[str, IsaInfo]): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -61,7 +61,7 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxC solution.update(perm) # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, [currentIsa], cxxCompiler, isaInfoMap) + solutionObject = Solution(solution, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, cxxCompiler, isaInfoMap) if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) @@ -77,13 +77,12 @@ def _getCustomKernelSolutionObj( internalSupportParams, cxxCompiler: str, debugConfig: DebugConfig, \ - currentIsa: IsaVersion, isaInfoMap: Dict[str, IsaInfo], directory=CUSTOM_KERNEL_PATH ): """Creates the Solution object for a custom kernel""" config = getCustomKernelConfig(kernelName, internalSupportParams, directory) - return Solution(config, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, [currentIsa], cxxCompiler, isaInfoMap) + return Solution(config, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, cxxCompiler, isaInfoMap) def _generateCustomKernelSolutions( @@ -93,14 +92,13 @@ def _generateCustomKernelSolutions( failOnMismatch, cxxCompiler: str, debugConfig: DebugConfig, - currentIsa: IsaVersion, isaInfoMap: Dict[str, IsaInfo] ): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, currentIsa, debugConfig, isaInfoMap) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, debugConfig, isaInfoMap) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -131,10 +129,24 @@ def _generateCustomKernelSolutions( return solutions -def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ - biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path, - useShortNames: bool, debugConfig: DebugConfig, currentIsa: IsaVersion, deviceId: int, isaInfoMap: Dict[str, IsaInfo]): +def writeBenchmarkFiles( + stepBaseDir, + solutions, + problemSizes, + biasTypeArgs, + factorDimArgs, + activationArgs, + icacheFlushArgs, + stepName, + solutionSummationSizes, + asmToolchain: AssemblyToolchain, + srcToolchain: SourceToolchain, + sourcePath: Path, + useShortNames: bool, + debugConfig: DebugConfig, + deviceId: int, + isaInfoMap: Dict[str, IsaInfo] + ): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -167,19 +179,26 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, - asmToolchain.assemblerVersion, debugConfig, ) # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ - sourcePath, asmToolchain, srcToolchain, \ - solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, debugConfig.splitGSU, \ - [isaToGfx(currentIsa)], isaInfoMap, errorTolerant=True, \ - generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], \ - useShortNames=useShortNames) + sourcePath, + asmToolchain, + srcToolchain, + solutions, + kernels, + kernelHelperObjs, + kernelWriterAssembly, + debugConfig.splitGSU, + [isaToGfx(isa) for isa in isaInfoMap.keys()], + errorTolerant=True, + generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], # put in debug config + compress=False, + useShortNames=useShortNames + ) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') @@ -189,7 +208,6 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ asmToolchain.assembler, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, - [currentIsa], isaInfoMap, ) newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) @@ -237,7 +255,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, - debugConfig: DebugConfig, currentIsa: IsaVersion, deviceId: int, isaInfoMap: Dict[str, IsaInfo] + debugConfig: DebugConfig, deviceId: int, isaInfoMap: Dict[str, IsaInfo] ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -318,12 +336,12 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize maxPossibleSolutions = len(forkPermutations) regSolutions = _generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler, \ - debugConfig, currentIsa, isaInfoMap) + benchmarkStep.constantParams, forkPermutations, str(srcToolchain.compiler.path), \ + debugConfig, isaInfoMap) kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, srcToolchain.compiler, debugConfig, \ - currentIsa, isaInfoMap) + not benchmarkStep.customKernelWildcard, str(srcToolchain.compiler.path), debugConfig, \ + isaInfoMap) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions @@ -352,7 +370,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath, useShortNames, debugConfig, currentIsa, deviceId, isaInfoMap) + sourcePath, useShortNames, debugConfig, deviceId, isaInfoMap) # ^ this mutates solutions # write cache data @@ -427,12 +445,11 @@ def main( buildTmpPath: Path, useShortNames: bool, debugConfig: DebugConfig, - currentIsa: IsaVersion, deviceId: int, isaInfoMap: Dict[str, IsaInfo] ): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" - ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) + ClientExecutable.getClientExecutable(str(srcToolchain.compiler.path), cCompiler, outputPath) if config is None: print(f'No config specified in {globalParameters["ConfigPath"]}, built client only') @@ -481,7 +498,6 @@ def main( benchmarkProblemsPath, useShortNames, debugConfig, - currentIsa, deviceId, isaInfoMap ) diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index f79e577a86..1d2f6d8f39 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -24,10 +24,10 @@ import subprocess from functools import lru_cache -from typing import Tuple +from typing import List, Dict from .Architectures import isaToGfx -from .Types import IsaVersion +from .Types import IsaVersion, IsaInfo def _tryAssembler( @@ -236,3 +236,13 @@ def initAsmBugs(asmCaps) -> dict: rv["ExplicitNC"] = asmCaps["HasExplicitNC"] return rv + +def makeIsaInfoMap(targetIsas: List[IsaVersion], cxxCompiler: str) -> Dict[IsaVersion, IsaInfo]: + isaInfoMap = {} + for v in targetIsas: + asmCaps = initAsmCaps(v, cxxCompiler, False) + archCaps = initArchCaps(v) + regCaps = initRegisterCaps(v, archCaps) + asmBugs = initAsmBugs(asmCaps) + isaInfoMap[v] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) + return isaInfoMap \ No newline at end of file diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index ccea7183d6..6cf25dfc7f 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -202,7 +202,7 @@ # False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed #) globalParameters["AsanBuild"] = False # build with asan -globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels +#globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp # debug for assembly @@ -1492,7 +1492,7 @@ def capRow(isaInfoMap, cap, capType): printTable([headerRow] + asmCapRows + archCapRows) -def assignGlobalParameters(config, targetIsas: List[IsaVersion], cxxCompiler=None): +def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo], cxxCompiler=None): """ Assign Global Parameters Each global parameter has a default parameter, and the user @@ -1571,21 +1571,10 @@ def assignGlobalParameters(config, targetIsas: List[IsaVersion], cxxCompiler=Non if "CodeObjectVersion" in config: globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - isaInfoMap = {} - for v in targetIsas: - asmCaps = initAsmCaps(v, cxxCompiler, False) - archCaps = initArchCaps(v) - regCaps = initRegisterCaps(v, archCaps) - asmBugs = initAsmBugs(asmCaps) - isaInfoMap[v] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) - if verbosity >= 1: printCapabilitiesTable(isaInfoMap) - isaList = list( - [v for v in targetIsas if isaInfoMap[v].asmCaps["SupportedISA"]] - ) - + isaList = list(isaInfoMap.keys()) validParameters["ISA"] = [IsaVersion(0, 0, 0), *isaList] # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future @@ -1614,6 +1603,10 @@ def assignGlobalParameters(config, targetIsas: List[IsaVersion], cxxCompiler=Non # The following keys may be present in the config, but are not (or no longer) global parameters. ignoreKeys = [ + "Architecture", + "ShortNames", + "PrintLevel", + "Device", "UseCompression", "CxxCompiler", "CCompiler", @@ -1633,8 +1626,6 @@ def assignGlobalParameters(config, targetIsas: List[IsaVersion], cxxCompiler=Non printWarning("Global parameter %s = %s unrecognised." % (key, value)) globalParameters[key] = value - return isaInfoMap - def setupRestoreClocks(): import atexit diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index cbc2fbc2a7..551a3d2f30 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -659,11 +659,11 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: IsaVersion, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo]): - return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap, solution.srcName) + def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo]): + return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, solution.srcName) @classmethod - def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: IsaVersion, cxxCompiler, isaInfoMap, srcName = "", deviceInfo=None): + def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, cxxCompiler, isaInfoMap, srcName = "", deviceInfo=None): rv = cls() if 'SolutionNameMin' in d: @@ -707,7 +707,7 @@ def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap, srcName) + rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcName) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index a05873f4ce..88bc088128 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,8 +36,9 @@ from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule -from .Common import globalParameters, printWarning, roundUp, print2, DebugConfig, DataDirection, SemanticVersion, \ - INDEX_CHARS, MAX_FILENAME_LENGTH, IsaVersion, IsaInfo +from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ + INDEX_CHARS, MAX_FILENAME_LENGTH, IsaVersion +from Tensile.Toolchain.Component import Assembler import abc import os @@ -362,14 +363,12 @@ def __init__( self, kernelMinNaming, kernelSerialNaming, - assembler: str, - amdClangVersion: SemanticVersion, + assembler: Assembler, debugConfig: DebugConfig, ): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler - self.amdClangVersion = amdClangVersion # this is a bug self.ti = None self.debugConfig = debugConfig @@ -2915,7 +2914,7 @@ def _initKernel(self, kernel, tensorParametersA, tensorParametersB): version = tuple(kernel["ISA"]) if self.ti == None: self.ti = TensileInstructions() - self.ti.init(version, self.assembler) + self.ti.init(version, str(self.assembler.path)) self.ti.setKernelInfo(version, kernel["WavefrontSize"]) self.ti.getArchCaps @@ -4992,29 +4991,6 @@ def _shortenFileBase(self, kernel): return firstPart + secondPart - def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): - kernelName = self.getKernelFileBase(useShortNames, kernel) - with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: - if not (self.amdClangVersion.major >= 6 and self.amdClangVersion.patch >= 32650): - code = [] - for line in f.readlines(): - if "amdhsa_user_sgpr_kernarg_preload" not in line: - code.append(line) - code = "".join(code) - else: - code = f.read() - - self.tPA = {} - self.tPB = {} - self.states.kernel = kernel - self.states.language = "ASM" - # we already do this in the solution ctor - assert "ISA" in kernel - self.states.version = tuple(kernel["ISA"]) - assert globalParameters["AsmCaps"][self.states.version]["SupportedISA"] - - return code - def _getKernelSource(self, kernel: Solution): """ Returns the source of the kernel, either C++ or assembly. diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index db61bae553..9e1233ee50 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -50,6 +50,7 @@ from .CustomKernels import isCustomKernelConfig from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, \ SemanticVersion, IsaVersion, IsaInfo +from Tensile.Toolchain.Component import Assembler from math import ceil, log, floor from copy import deepcopy @@ -86,13 +87,37 @@ def __init__( self, kernelMinNaming, kernelSerialNaming, - assembler: str, - amdClangVersion: SemanticVersion, + assembler: Assembler, debugConfig: DebugConfig, ): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion, debugConfig) + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, debugConfig) + + + def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): + kernelName = self.getKernelFileBase(useShortNames, kernel) + with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: + rocmVersion = self.assembler.rocm_version + if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650): + code = [] + for line in f.readlines(): + if "amdhsa_user_sgpr_kernarg_preload" not in line: + code.append(line) + code = "".join(code) + else: + code = f.read() + + self.tPA = {} + self.tPB = {} + self.states.kernel = kernel + self.states.language = "ASM" + self.states.version = kernel["ISA"] + + return code + - def getSourceFileString(self, kernel, useShortNames: bool=False) -> Tuple[int, str]: + def getSourceFileString(self, + kernel, + useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" # Skip if .o files will have already been built for this file if kernel.duplicate: @@ -11196,7 +11221,7 @@ def globalWriteBatch(self, kernel, tPA, tPB, activation, ss: StoreState, batchId addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, biasLocalBarrierInit, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, \ batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, packdata, self, factorDim, \ - self.amdClangVersion) + self.assembler.version) ############################################################################## def openPrefetchGlobalRead2(self, kernel): diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index b8380adac7..c53959b915 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -29,7 +29,7 @@ from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, IsaInfo, isaToGfx, INDEX_CHARS +from .Common import IsaInfo, isaToGfx, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 88c3f09515..b18361411f 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -226,7 +226,7 @@ def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool, printSolution # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, [solutionState["ISA"]], cxxCompiler, isaInfoMap, srcFile) + solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcFile) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) @@ -242,12 +242,38 @@ class LibraryLogic(NamedTuple): exactLogic: list library: SolutionLibrary.MasterSolutionLibrary -def parseLibraryLogicFile(filename, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, archs, isaInfoMap: Dict[str, IsaInfo]): +def parseLibraryLogicFile( + filename, + cxxCompiler, + splitGSU: bool, + printSolutionRejectionReason: bool, + archs, + isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool + ): """Wrapper function to read and parse a library logic file.""" - return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, splitGSU, printSolutionRejectionReason, archs, isaInfoMap) - - -def parseLibraryLogicData(data, srcFile, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, archs, isaInfoMap: Dict[str, IsaInfo]): + return parseLibraryLogicData( + read(filename, True), + filename, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + archs, + isaInfoMap, + lazyLibraryLoading + ) + + +def parseLibraryLogicData( + data, + srcFile, + cxxCompiler, + splitGSU: bool, + printSolutionRejectionReason: bool, + archs, + isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool + ): """Parses the data of a library logic file.""" if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) @@ -293,7 +319,7 @@ def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap, srcFile) + solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcFile) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -307,7 +333,15 @@ def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: solutions = [solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) for solutionState in data["Solutions"]] - newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap) + newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState( + data, + solutions, + splitGSU, + printSolutionRejectionReason, + cxxCompiler, + isaInfoMap, + lazyLibraryLoading + ) return LibraryLogic(data["ScheduleName"], data["ArchitectureName"], problemType, solutions, \ data.get("ExactLogic"), newLibrary) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 2c54ce81f4..445016f863 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -23,14 +23,13 @@ ################################################################################ import itertools -from typing import List, Dict +from typing import Dict from . import Properties from . import Hardware -from . import Common from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state, IsaVersion, IsaInfo +from .Common import state, IsaInfo, gfxToIsa class SingleSolutionLibrary: Tag = "Single" @@ -303,9 +302,9 @@ def FromOriginalState(cls, origSolutions, splitGSU: bool, printSolutionRejectionReason: bool, - supportedISA: List[IsaVersion], cxxCompiler, isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool, solutionClass=Contractions.Solution, libraryOrder=None, placeholderName='TensileLibrary'): @@ -319,7 +318,7 @@ def hardware(d, problemType, solutions, library, placeholderName): if devicePart == "fallback": pred = Hardware.HardwarePredicate("TruePred") else: - pred = Hardware.HardwarePredicate.FromHardware(Common.gfxToIsa(devicePart), cuCount) + pred = Hardware.HardwarePredicate.FromHardware(gfxToIsa(devicePart), cuCount) newLib.rows.append({"predicate": pred, "library": library}) @@ -402,7 +401,7 @@ def selection(d, problemType, solutions, library, placeholderName): else: assert 0 and "Unrecognized LibraryType." - if Common.globalParameters["LazyLibraryLoading"]: + if lazyLibraryLoading: placeholderName += '_' + str(problemType.aType) + str(problemType.bType) placeholderName += '_' + str(problemType.cType) + str(problemType.computeInputType) if problemType.activationType != 'none': @@ -447,7 +446,7 @@ def selection(d, problemType, solutions, library, placeholderName): # end library creation functions if libraryOrder is None: - if Common.globalParameters["LazyLibraryLoading"]: + if lazyLibraryLoading: libraryOrder = [ hardware, operationIdentifier, performanceMetric, predicates, placeholder, selection @@ -467,9 +466,9 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions, splitGSU, printSolutionRejectionReason, - supportedISA, cxxCompiler, isaInfoMap, + lazyLibraryLoading, solutionClass, libraryOrder[placeholderIndex:], placeholderName) @@ -477,7 +476,7 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -498,8 +497,8 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, supportedISA: List[IsaVersion], isaInfoMap): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, supportedISA, cxxCompiler, isaInfoMap) for s in solutions]) + def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap) for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index adec3a070d..f6c0a4f9eb 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -37,22 +37,14 @@ from .CustomKernels import isCustomKernelConfig from .AsmStoreState import VectorDataTypes -from .Common import assignParameterWithDefault, \ - defaultProblemType, defaultSolution, \ - defaultInternalSupportParams, \ - globalParameters, internalParameters, \ - print2, printExit, printWarning, \ - validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, \ - INDEX_CHARS, IsaVersion, print1 - from .Common import assignParameterWithDefault, IsaInfo, \ defaultProblemType, defaultSolution, \ defaultInternalSupportParams, \ globalParameters, internalParameters, \ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS, print1 + validGEMMTypes, HPATypes, roundUp, validWMMA, \ + INDEX_CHARS, IsaVersion from collections import OrderedDict from collections.abc import Mapping @@ -1066,9 +1058,8 @@ def __init__( config, splitGSU: bool, printSolutionRejectionReason: bool, - targetIsas: List[IsaVersion], cxxCompiler: str, - isaInfoMap: Dict[str, IsaInfo], + isaInfoMap: Dict[IsaVersion, IsaInfo], srcName: str = "" ): @@ -1078,6 +1069,7 @@ def __init__( self.srcName = srcName self.splitGSU = splitGSU config = config + targetIsas = list(isaInfoMap.keys()) self._state = {} # problem type diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 707575dcc3..8b4f693156 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -37,10 +37,10 @@ from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ - detectGlobalCurrentISA, IsaVersion, verbosity, IsaInfo + detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap from Tensile.KernelWriter import DebugConfig -from Tensile.Toolchain.Assembly import AssemblyToolchain -from Tensile.Toolchain.Source import SourceToolchain +from Tensile.Toolchain.Assembly import AssemblyToolchain, makeAssemblyToolchain +from Tensile.Toolchain.Source import SourceToolchain, makeSourceToolchain from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults from Tensile.Utilities.Decorators.Profile import profile from Tensile import BenchmarkProblems @@ -65,7 +65,6 @@ def executeStepsInConfig( isaInfoMap: Dict[str, IsaInfo], cCompiler: str, debugConfig: DebugConfig, - currentIsa: IsaVersion, deviceId: int ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -101,7 +100,6 @@ def executeStepsInConfig( buildTmpPath, config["ShortNames"], debugConfig, - currentIsa, deviceId, isaInfoMap, ) @@ -363,6 +361,8 @@ def makeDebugConfig(config: dict) -> DebugConfig: printSolutionRejectionReason = config["PrintSolutionRejectionReason"] if "SplitGSU" in config: splitGSU = config["SplitGSU"] + if "SaveTemps" in config: + splitGSU = config["SaveTemps"] return DebugConfig(enableAsserts, enableDebugA, @@ -482,12 +482,20 @@ def Tensile(userArgs): print(f"Successfully retrieve Max frequency: {max_frequency} for device {device_id}") store_max_frequency(max_frequency) - cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) + cxxCompiler, cCompiler, _, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) currentIsa = detectGlobalCurrentISA(device_id) - isaInfoMap = assignGlobalParameters(config.get("GlobalParameters", {}), [currentIsa], cxxCompiler) - - asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) - srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) + isaInfoMap = makeIsaInfoMap([currentIsa], cxxCompiler) + assignGlobalParameters(config.get("GlobalParameters", {}), isaInfoMap, cxxCompiler) + + asmToolchain = makeAssemblyToolchain( + cxxCompiler, + offloadBundler, + args.CodeObjectVersion, + ) + srcToolchain = makeSourceToolchain( + cxxCompiler, + offloadBundler, + ) overrideParameters = argUpdatedGlobalParameters(args) @@ -503,7 +511,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler, debugConfig, currentIsa, device_id) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler, debugConfig, device_id) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 5e48e0d27e..bbab2e8017 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -35,17 +35,16 @@ from Tensile.Common import ( CHeader, DebugConfig, - detectGlobalCurrentISA, gfxToIsa, HR, IsaInfo, IsaVersion, + makeIsaInfoMap, ParallelMap2, SemanticVersion, architectureMap, assignGlobalParameters, ensurePath, - globalParameters, isaToGfx, print1, print2, @@ -66,13 +65,13 @@ from Tensile.SolutionLibrary import MasterSolutionLibrary from Tensile.SolutionStructs import Solution from Tensile.TensileInstructions import TensileInstructions -from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles -from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles +from Tensile.Toolchain.Assembly import makeAssemblyToolchain, buildAssemblyCodeObjectFiles +from Tensile.Toolchain.Source import makeSourceToolchain, SourceToolchain, buildSourceCodeObjectFiles from Tensile.Toolchain.Validators import ( ToolchainDefaults, - getVersion, validateToolchain, ) +from Tensile.Toolchain.Component import Assembler from Tensile.Utilities.Decorators.Profile import profile from Tensile.Utilities.Decorators.Timing import timing @@ -182,9 +181,8 @@ def writeHelpers( kernelHeaderFile.write(CHeader) kernelSourceFile.write('#include "Kernels.h"\n') kernelHeaderFile.write("#pragma once\n") - if globalParameters["RuntimeLanguage"] == "HIP": - kernelHeaderFile.write("#include \n") - kernelHeaderFile.write("#include \n\n") + kernelHeaderFile.write("#include \n") + kernelHeaderFile.write("#include \n\n") kernelHeaderFile.write('#include "KernelHeader.h"\n\n') HeaderText = "" for ko in kernelHelperObjs: @@ -207,7 +205,6 @@ def writeSolutionsAndKernels( kernelWriterAssembly, splitGSU: bool, cmdlineArchs: List[str], - isaInfoMap: Dict[str, IsaInfo], errorTolerant=False, generateSourcesAndExit=False, compress=True, @@ -252,7 +249,7 @@ def writeSolutionsAndKernels( def assemble(ret): p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + asmToolchain.assembler(isaToGfx(isa), wavefrontsize, str(p), str(p.with_suffix(".o"))) unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) @@ -269,10 +266,18 @@ def assemble(ret): if not generateSourcesAndExit: codeObjectFiles += buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames + asmToolchain.linker, + asmToolchain.bundler, + asmKernels, + kernelWriterAssembly, + destLibPath, + assemblyTmpPath, + compress, + useShortNames ) buildSourceCodeObjectFiles( - srcToolchain, + srcToolchain.compiler, + srcToolchain.bundler, destLibPath, objectTmpPath, outputPath, @@ -291,7 +296,6 @@ def writeSolutionsAndKernelsTCL( kernelHelperObjs, kernelWriterAssembly, cmdlineArchs: List[str], - isaInfoMap: Dict[str, IsaInfo], compress=True, useShortNames=False, ): @@ -323,7 +327,7 @@ def writeSolutionsAndKernelsTCL( def assemble(ret): p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + asmToolchain.assembler(isaToGfx(isa), wavefrontsize, str(p), str(p.with_suffix(".o"))) unaryProcessKernelSource = functools.partial( processKernelSource, kernelWriterAssembly, TensileInstructions(), useShortNames @@ -337,20 +341,28 @@ def assemble(ret): multiArg=False, return_as="list" ) + buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress, useShortNames + asmToolchain.linker, + asmToolchain.bundler, + asmKernels, + kernelWriterAssembly, + destLibPath, + assemblyTmpPath, + compress, + useShortNames ) writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" buildSourceCodeObjectFiles( - srcToolchain, + srcToolchain.compiler, + srcToolchain.bundler, destLibPath, objectTmpPath, outputPath, srcKernelFile, cmdlineArchs, - isaInfoMap, ) return len(uniqueAsmKernels) @@ -360,8 +372,7 @@ def assemble(ret): def getSolutionAndKernelWriters( solutions, kernels, - assembler: str, - assemblerVersion: SemanticVersion, + assembler: Assembler, ): kernelSerialNaming = Solution.getSerialNaming(kernels) solutionMinNaming = Solution.getMinNaming(solutions) @@ -369,8 +380,7 @@ def getSolutionAndKernelWriters( kernelWriterAssembly = KernelWriterAssembly( kernelMinNaming, kernelSerialNaming, - assembler, - assemblerVersion, + assembler, DebugConfig(), ) @@ -441,10 +451,11 @@ def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler, isaInfoMap): fIter = zip( logicFiles, itertools.repeat(cxxCompiler), - itertools.repeat(isaInfoMap), itertools.repeat(splitGSU), itertools.repeat(printSolutionRejectionReason), itertools.repeat(archs), + itertools.repeat(isaInfoMap), + itertools.repeat(args["LazyLibraryLoading"]), ) def libraryIter(lib: MasterSolutionLibrary): @@ -536,40 +547,39 @@ def run(): global verbosity verbosity = arguments["PrintLevel"] outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) - cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( + cxxCompiler, _, offloadBundler, _, _ = validateToolchain( arguments["CxxCompiler"], arguments["CCompiler"], arguments["OffloadBundler"], arguments["Assembler"], ToolchainDefaults.HIP_CONFIG, ) - print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") - print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") - print1(f"# C Compiler: {cCompiler} (version {getVersion(cCompiler)})") - print1(f"# Assembler: {assembler} (version {getVersion(assembler)})") - print1(f"# Offload Bundler: {offloadBundler} (version {getVersion(offloadBundler)})") - print1(f"# Code Object Version: {arguments['CodeObjectVersion']}") - print1(f"# Architecture(s): {arguments['Architecture']}") - print1(f"# Library Format: {arguments['LibraryFormat']}") if ";" in arguments["Architecture"]: archs = arguments["Architecture"].split(";") else: archs = arguments["Architecture"].split("_") targetIsas = [gfxToIsa(a) for a in archs] - isaInfoMap = assignGlobalParameters(arguments, targetIsas, cxxCompiler) + isaInfoMap = makeIsaInfoMap(targetIsas, cxxCompiler) + assignGlobalParameters(arguments, isaInfoMap, cxxCompiler) - asmToolchain = AssemblyToolchain( - assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"] + asmToolchain = makeAssemblyToolchain( + cxxCompiler, + offloadBundler, + arguments["CodeObjectVersion"], + arguments["BuildIdKind"] ) - srcToolchain = SourceToolchain( + srcToolchain = makeSourceToolchain( cxxCompiler, offloadBundler, - globalParameters["BuildIdKind"], - globalParameters["AsanBuild"], - globalParameters["SaveTemps"], + arguments["AsanBuild"], + arguments["BuildIdKind"], + save_temps=False ) + print1(asmToolchain.assembler) + print1(asmToolchain.bundler) + if not os.path.exists(arguments["LogicPath"]): printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") @@ -616,7 +626,6 @@ def validLogicFile(p: Path): for logicFile in logicFiles: print2("# %s" % logicFile) - currentIsa = detectGlobalCurrentISA(0) solutions, masterLibraries = generateLogicDataAndSolutions( logicFiles, arguments, cxxCompiler, isaInfoMap @@ -624,7 +633,7 @@ def validLogicFile(p: Path): kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion + solutions, kernels, asmToolchain.assembler ) copyStaticFiles(outputPath) @@ -637,7 +646,6 @@ def validLogicFile(p: Path): kernelHelperObjs, kernelWriterAssembly, archs, - isaInfoMap, useShortNames=arguments["ShortNames"], compress=arguments["UseCompression"], ) @@ -651,7 +659,7 @@ def validLogicFile(p: Path): splitGSU = False for archName, newMasterLibrary in masterLibraries.items(): if archName in archs: - if globalParameters["LazyLibraryLoading"]: + if arguments["LazyLibraryLoading"]: masterFile = os.path.join(newLibraryDir, "TensileLibrary_lazy_" + archName) else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_" + archName) @@ -662,7 +670,7 @@ def validLogicFile(p: Path): lib.applyNaming(splitGSU, kernelMinNaming) LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) - if not globalParameters["KeepBuildTmp"]: + if not arguments["KeepBuildTmp"]: buildTmp = Path(arguments["OutputPath"]).parent / "library" / "build_tmp" if buildTmp.exists() and buildTmp.is_dir(): shutil.rmtree(buildTmp) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 101aff3363..12176af594 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -24,126 +24,29 @@ import collections import math -import os -import shlex import shutil import subprocess from pathlib import Path -from typing import List, Union +from typing import List, Union, NamedTuple -from ..Common import globalParameters, print2, ensurePath, SemanticVersion, isaToGfx +from ..Common import globalParameters, print2, ensurePath, isaToGfx from ..KernelWriterAssembly import KernelWriterAssembly -from ..Toolchain.Validators import getVersion from ..SolutionStructs import Solution -class AssemblyToolchain: - def __init__(self, assembler: str, bundler: str, buildIdKind: str, coVersion: str): - self.assembler = assembler - self.assemblerVersion = SemanticVersion(*[int(c) for c in getVersion(assembler).split(".")[:3]]) - self.bundler = bundler - self.buildIdKind = buildIdKind - self.coVersion = coVersion - - def invoke(self, args: List[str], desc: str=""): - """Invokes a subprocess with the provided arguments. - - Args: - args: A list of arguments to pass to the subprocess. - desc: A description of the subprocess invocation. - - Raises: - RuntimeError: If the subprocess invocation fails. - """ - print2(f"{desc}: {' '.join(args)}") - try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"Error with {desc}: {err.output}\n" - f"Failed command: {' '.join(args)}" - ) - print2(f"Output: {out}") - return out - - def assemble(self, srcPath: str, destPath: str, gfx: str, wavefrontSize: int, debug: bool=False): - """Assemble an assembly source file into an object file. - - Args: - srcPath: The path to the assembly source file. - destPath: The destination path for the generated object file. - coVersion: The code object version to use. - isa: The target GPU architecture in ISA format. - wavefrontSize: The wavefront size to use. - """ - launcher = shlex.split(os.environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')) - args = [ - *launcher, - self.assembler, - "-x", "assembler", - "--target=amdgcn-amd-amdhsa", - f"-mcode-object-version={self.coVersion}", - f"-mcpu={gfx}", - "-mwavefrontsize64" if wavefrontSize == 64 else "-mno-wavefrontsize64" - "-g" if debug else "", - "-c", - "-o", destPath, srcPath - ] - - return self.invoke(args, "Assembling assembly source code into object file (.s -> .o)") - - def link(self, srcPaths: List[str], destPath: str): - """Links object files into a code object file. - - Args: - srcPaths: A list of paths to object files. - destPath: A destination path for the generated code object file. - - Raises: - RuntimeError: If linker invocation fails. - """ - if os.name == "nt": - # Use args file on Windows b/c the command may exceed the limit of 8191 characters - with open(Path.cwd() / "clang_args.txt", "wt") as file: - file.write(" ".join(objFiles)) - file.flush() - args = [ - self.assembler, - "--target=amdgcn-amd-amdhsa", - "-o", destPath, "@clang_args.txt"] - else: - args = [ - self.assembler, - "--target=amdgcn-amd-amdhsa", - "-Xlinker", f"--build-id={self.buildIdKind}", - "-o", destPath, *srcPaths - ] - - return self.invoke(args, "Linking assembly object files into code object (*.o -> .co)") - - def compress(self, srcPath: str, destPath: str, gfx: str): - """Compresses a code object file using the provided bundler. - - Args: - srcPath: The source path of the code object file to be compressed. - destPath: The destination path for the compressed code object file. - gfx: The target GPU architecture. - - Raises: - RuntimeError: If compressing the code object file fails. - """ - args = [ - self.bundler, - "--compress", - "--type=o", - "--bundle-align=4096", - f"--targets=host-x86_64-unknown-linux-gnu,hipv4-amdgcn-amd-amdhsa-unknown-{gfx}", - "--input=/dev/null", - f"--input={srcPath}", - f"--output={destPath}", - ] - - return self.invoke(args, "Bundling/compressing code object file (.co -> .co)") +from .Component import Assembler, Linker, Bundler + +class AssemblyToolchain(NamedTuple): + assembler: Assembler + linker: Linker + bundler: Bundler + + +def makeAssemblyToolchain(assembler_path, bundler_path, co_version, build_id_kind="sha1"): + compiler = Assembler(assembler_path, co_version) + linker = Linker(assembler_path, build_id_kind) + bundler = Bundler(bundler_path) + return AssemblyToolchain(compiler, linker, bundler) def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: @@ -169,8 +72,10 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF return newObjFilesOutput + def buildAssemblyCodeObjectFiles( - toolchain: AssemblyToolchain, + linker: Linker, + bundler: Bundler, kernels: List[Solution], writer: KernelWriterAssembly, destDir: Union[Path, str], @@ -219,10 +124,10 @@ def buildAssemblyCodeObjectFiles( coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(useShortNames, kernel) + extObj))) for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(objFiles, coFileRaw) - toolchain.link(objFiles, str(coFileRaw)) + linker(objFiles, str(coFileRaw)) coFile = destDir / coFileRaw.name.replace(extCoRaw, extCo) if compress: - toolchain.compress(str(coFileRaw), str(coFile), gfx) + bundler.compress(str(coFileRaw), str(coFile), gfx) else: shutil.move(coFileRaw, coFile) coFiles.append(coFile) diff --git a/tensilelite/Tensile/Toolchain/Component.py b/tensilelite/Tensile/Toolchain/Component.py new file mode 100644 index 0000000000..06a6c653c0 --- /dev/null +++ b/tensilelite/Tensile/Toolchain/Component.py @@ -0,0 +1,339 @@ + +from os import name as os_name +from os import environ +from pathlib import Path +from re import search, IGNORECASE +from shlex import split +from subprocess import check_output, STDOUT, CalledProcessError, PIPE, run +from typing import List + +from Tensile.Common import SemanticVersion, print1 +from .Validators import ToolchainDefaults + +def _invoke(args: List[str], desc: str=""): + """Invokes a command with the provided arguments in a subprocess. + Args: + args: A list of arguments to pass to the subprocess. + desc: A description of the subprocess invocation. + Raises: + RuntimeError: If the subprocess invocation fails. + Return: + subprocess output + """ + #print1(f"{desc}: {' '.join(args)}") + try: + out = check_output(args, stderr=STDOUT) + except CalledProcessError as err: + raise RuntimeError( + f"Error with {desc}: {err.output}\n" + f"Failed command: {' '.join(args)}" + ) + #print2(f"Output: {out}") + return out + + +def _getVersion(executable: str, versionFlag: str, regex: str) -> str: + """Compute the version string of a toolchain component. + + Args: + executable: The toolchain component to check the version of. + versionFlag: The flag to pass to the executable to get the version. + regex: pattern used to extract version string. + Raises: + RuntimeError: If querying executable for version fails. + Return: + Executable version + """ + args = f'"{executable}" "{versionFlag}"' + try: + output = run(args, stdout=PIPE, shell=True).stdout.decode().strip() + match = search(regex, output, IGNORECASE) + result = match.group(1) if match else "" + return SemanticVersion(*[int(c.split("-")[0]) for c in result.split(".")[:3]]) + except Exception as e: + raise RuntimeError(f"Failed to get version when calling {args}: {e}") + + +def get_rocm_version() -> str: + """Compute the ROCm version string using hipconfig. + + Raises: + RuntimeError: If hipconfig fails to execute. + Return: + ROCm version string + """ + return _getVersion(ToolchainDefaults.HIP_CONFIG, "--version", r'(.+)') + + +class Component: + """A class used to represent a ROCm toolchain component such as clang++""" + + _rocm_version = get_rocm_version() + + def __init__(self, component_path: Path, version_flag: str="--version", version_regex: str=r"version\s+([\d.]+)"): + self._version = _getVersion(str(component_path), version_flag, version_regex) + self._component_path = component_path + + def __str__(self): + result = f"ROCm {Component._rocm_version.major}.{Component._rocm_version.minor}.{Component._rocm_version.patch} " + result += f"Component path: {self._component_path} version: {self._version.major}.{self._version.minor}.{self._version.patch}" + return result + + @property + def path(self): + return self._component_path + + @property + def version(self): + return self._version + + @property + def rocm_version(self): + return Component.rocm_version + + +class Assembler(Component): + """ + ROCm assembler class used to build objects from assembly source files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + path : str + path to assembler + + Methods + ------- + __call__(self, targetGfx: str, wavefrontSize: int, debug: bool, srcPath: str, destPath: str) + Invokes the assembler on the provided arguments + """ + + def __init__(self, component_path: Path, co_version: str, debug: bool=False): + """Constructs instance of assmebler. + + Args: + assembler_path: The path to the assember. + co_version: The code object version to use. + """ + + super(Assembler, self).__init__(component_path) + + + self._default_args = [ + *split(environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')), + str(component_path), + "-x", "assembler", + "--target=amdgcn-amd-amdhsa", + "-g" if debug else "", + f"-mcode-object-version={co_version}", + "-c", + ] + + def __call__(self, targetGfx: str, wavefrontSize: int, srcPath: str, destPath: str): + """Assemble an assembly source file into an object file. + Args: + targetGfx: The target GPU gfx architecture. + wavefrontSize: The wavefront size to use. + debug: add debug flags if True. + srcPath: The path to the assembly source file. + destPath: The destination path for the generated object file. + """ + args = self._default_args + args = [ + *args, + f"-mcpu={targetGfx}", + "-mwavefrontsize64" if wavefrontSize == 64 else "-mno-wavefrontsize64", + srcPath, + "-o", + destPath + ] + return _invoke(args, "Assembling assembly source code into object file (.s -> .o)") + + +class Compiler(Component): + """ + ROCm compiler class used to build objects from C++ source files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + path : str + path to compiler + + Methods + ------- + __call__(self, include_path: str, target_list: List[str], srcPath: str, destPath: str): + Invokes the compiler on the provided arguments + """ + + def __init__(self, compiler_path: Path, build_id_kind: str, asan_build: bool=False, save_temps: bool=False): + """Constructs and instance of a Compiler.""" + super(Compiler, self).__init__(compiler_path) + + self.default_args = [ + *split(environ.get("Tensile_CXX_COMPILER_LAUNCHER", "")), + compiler_path, + "-D__HIP_HCC_COMPAT_MODE__=1", + "--offload-device-only", + "-x", "hip", "-O3", + "-Xoffload-linker", f"--build-id={build_id_kind}", + "-std=c++17", + ] + + if asan_build: + self.default_args.extend(["-fsanitize=address", "-shared-libasan", "-fuse-ld=lld"]) + if save_temps: + self.default_args.append("--save-temps") + if os_name == "nt": # should we use fPIIC on all arches? + self.default_args.extend(["-fms-extensions", "-fms-compatibility", "-fPIC", "-Wno-deprecated-declarations"]) + + + def __call__(self, include_path: str, target_list: List[str], srcPath: str, destPath: str): + """Compiles a source file into an object file. + + Args: + include_path: Path appened to "-I" to directory with required include files. + target_list: List of offload architectures of the form gfxXYZ e.g. gfx942. + sercPath: The path to the source file. + destPath: Path to the object file output during compilation. + Raises: + RuntimeError: If the compilation command fails. + """ + archFlags = [f"--offload-arch={gfx}" for gfx in target_list] + args = [ + *(self.default_args), "-I", include_path, *archFlags, srcPath, "-c", "-o", destPath + ] + return _invoke(args, f"Compiling HIP source kernels into objects (.cpp -> .o)") + + +class Bundler(Component): + """ + ROCm bundler class used to unbundle objects into code object files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + Methods + ------- + __call__(self, targetGfx: str, wavefrontSize: int, debug: bool, srcPath: str, destPath: str) + Invokes the assembler on the provided arguments + def targets(self, objFile: str): + returns a list of target triple strings of the form amdgcn-amd--gfx942 + def compress(self, srcPath: str, destPath: str, target: str): + Compresses a code object file using the provided bundler. + """ + + def __init__(self, bundler_path: Path): + """Constructs and instance of a Bunder.""" + super(Bundler, self).__init__(bundler_path) + + def targets(self, objFile: str): + """returns a list of target triple strings of the form amdgcn-amd--gfx942""" + args = [self._component_path, "--type=o", f"--input={objFile}", "-list"] + return _invoke(args, f"Listing target triples in object file").decode().split("\n") + + def compress(self, srcPath: str, destPath: str, target: str): + """Compresses a code object file using the provided bundler. + + Args: + srcPath: The source path of the code object file to be compressed. + destPath: The destination path for the compressed code object file. + gfx: The target GPU architecture. + + Raises: + RuntimeError: If compressing the code object file fails. + """ + args = [ + self._component_path, + "--compress", + "--type=o", + "--bundle-align=4096", + f"--targets=host-x86_64-unknown-linux-gnu,hipv4-amdgcn-amd-amdhsa-unknown-{target}", + "--input=/dev/null", + f"--input={srcPath}", + f"--output={destPath}", + ] + + return _invoke(args, "Bundling/compressing code object file (.co -> .co)") + + def __call__(self, target: str, srcPath: str, destPath: str): + """Unbundles source code object files using the Clang Offload Bundler. + Args: + target: The target triple, see https://llvm.org/docs/AMDGPUUsage.html#target-triples. + srcPath: The path to the input object file. + destPath: The path to the unbundled code object. + Raises: + RuntimeError: If unbundling the source code object file fails. + """ + args = [ + self._component_path, + "--type=o", + f"--targets={target}", + f"--input={srcPath}", + f"--output={destPath}", + "--unbundle", + ] + return _invoke(args, f"Unbundling source code object file") + + +class Linker(Component): + """ + ROCm Linker class used to link objects into code object files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + Methods + ------- + __call__(self, srcPaths: List[str], destPath: str): + Invokes the linker on the provided arguments + """ + + def __init__(self, linker_path: Path, build_id_kind: str): + """Constructs and instance of a Linker.""" + super(Linker, self).__init__(linker_path) + self.default_args = [ + self._component_path, + "--target=amdgcn-amd-amdhsa", + "-Xlinker", f"--build-id={build_id_kind}", + ] + + + def __call__(self, srcPaths: List[str], destPath: str): + """Links object files into a code object file. + + Args: + srcPaths: A list of paths to object files. + destPath: A destination path for the generated code object file. + Raises: + RuntimeError: If linker invocation fails. + """ + if os_name == "nt": + # Use args file on Windows b/c the command may exceed the limit of 8191 characters + with open(Path.cwd() / "clang_args.txt", "wt") as file: + file.write(" ".join(srcPaths)) + file.flush() + args = [*(self.default_args), "-o", destPath, "@clang_args.txt"] + else: + args = [*(self.default_args), *srcPaths, "-o", destPath] + return _invoke(args, "Linking assembly object files into code object (*.o -> .co)") diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 160ea359df..b572be3e1b 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -24,118 +24,25 @@ import os import re -import shlex import shutil -import subprocess from pathlib import Path from timeit import default_timer as timer -from typing import List, Union, Dict - -from ..Common import globalParameters, print1, print2, ensurePath - -class SourceToolchain: - def __init__(self, compiler: str, bundler: str, buildIdKind: str, asanBuild: bool=False, saveTemps: bool=False): - self.compiler = compiler - self.bundler = bundler - self.buildIdKind = buildIdKind - self.asanBuild = asanBuild - self.saveTemps = saveTemps - - def invoke(self, args: List[str], desc: str=""): - """Invokes a subprocess with the provided arguments. - - Args: - args: A list of arguments to pass to the subprocess. - desc: A description of the subprocess invocation. - - Raises: - RuntimeError: If the subprocess invocation fails. - """ - print2(f"{desc}: {' '.join(args)}") - try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"Error with {desc}: {err.output}\n" - f"Failed command: {' '.join(args)}" - ) - print2(f"Output: {out}") - return out - - def compile(self, srcPath: str, destPath: str, includePath: str, gfxs: List[str]): - """Compiles a source file into an object file. - - Args: - cmdlineArchs: List of architectures for offloading. - kernelFile: The path to the kernel source file. - buildPath: The build directory path. - objectFilename: The name of the output object file. - outputPath: The output directory path. - globalParameters: A dictionary of global parameters. - - Raises: - RuntimeError: If the compilation command fails. - """ - launcher = shlex.split(os.environ.get("Tensile_CXX_COMPILER_LAUNCHER", "")) - - hipFlags = [ - "-D__HIP_HCC_COMPAT_MODE__=1", - "--offload-device-only", - "-x", "hip", "-O3", - "-I", includePath, - "-Xoffload-linker", f"--build-id={self.buildIdKind}", - "-std=c++17", - ] - if self.asanBuild: - hipFlags.extend(["-fsanitize=address", "-shared-libasan", "-fuse-ld=lld"]) - if self.saveTemps: - hipFlags.append("--save-temps") - if os.name == "nt": - hipFlags.extend(["-fms-extensions", "-fms-compatibility", "-fPIC", "-Wno-deprecated-declarations"]) - - archFlags = [f"--offload-arch={gfx}" for gfx in gfxs] - - args = [ - *launcher, self.compiler, *hipFlags, *archFlags, srcPath, "-c", "-o", destPath - ] - - return self.invoke(args, f"Compiling HIP source kernels into objects (.cpp -> .o)") - - - def targets(self, objFile: str): - """Lists the target triples in an object file. - - Args: - objFile: The object file path. - - Returns: - List of target triples in the object file. - """ - args = [self.bundler, "--type=o", f"--input={objFile}", "-list"] - return self.invoke(args, f"Listing target triples in object file").decode().split("\n") - - def unbundle(self, target: str, srcPath: str, destPath: str): - """Unbundles source code object files using the Clang Offload Bundler. - - Args: - target: The target triple, see https://llvm.org/docs/AMDGPUUsage.html#target-triples. - infile: The path to the input object file. - outfileRaw: The path to the unbundled code object. - - Raises: - RuntimeError: If unbundling the source code object file fails. - """ - args = [ - self.bundler, - "--type=o", - f"--targets={target}", - f"--input={srcPath}", - f"--output={destPath}", - "--unbundle", - ] - - return self.invoke(args, f"Unbundling source code object file") +from typing import List, Union, NamedTuple + +from ..Common import globalParameters, print1, ensurePath + +from .Component import Compiler, Bundler + +class SourceToolchain(NamedTuple): + compiler: Compiler + bundler: Bundler + + +def makeSourceToolchain(compiler_path, bundler_path, asan_build=False, build_id_kind="sha1", save_temps=False): + compiler = Compiler(compiler_path, build_id_kind, asan_build, save_temps) + bundler = Bundler(bundler_path) + return SourceToolchain(compiler, bundler) def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Path, str], arch: str) -> Union[Path, None]: @@ -165,7 +72,8 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa def buildSourceCodeObjectFiles( - toolchain: SourceToolchain, + compiler: Compiler, + bundler: Bundler, destDir: Union[Path, str], tmpObjDir: Union[Path, str], includeDir: Union[Path, str], @@ -198,15 +106,15 @@ def buildSourceCodeObjectFiles( coPaths= [] objPath = str(tmpObjDir / objFilename) - toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) + compiler(str(includeDir), cmdlineArchs, str(kernelPath), objPath) - for target in toolchain.targets(objPath): + for target in bundler.targets(objPath): match = re.search("gfx.*$", target) if match: arch = re.sub(":", "-", match.group()) coPathRaw = _computeSourceCodeObjectFilename(target, kernelPath.stem, tmpObjDir, arch) if not coPathRaw: continue - toolchain.unbundle(target, objPath, str(coPathRaw)) + bundler(target, objPath, str(coPathRaw)) coPath = str(destDir / coPathRaw.stem) coPathsRaw.append(coPathRaw) diff --git a/tensilelite/Tensile/Toolchain/Validators.py b/tensilelite/Tensile/Toolchain/Validators.py index 5c2381a652..7920fa96f4 100644 --- a/tensilelite/Tensile/Toolchain/Validators.py +++ b/tensilelite/Tensile/Toolchain/Validators.py @@ -246,19 +246,3 @@ def validateToolchain(*args: str): out = (_validateExecutable(x, searchPaths) for x in args) return next(out) if len(args) == 1 else tuple(out) - - -def getVersion(executable: str, versionFlag: str="--version", regex: str=r"version\s+([\d.]+)") -> str: - """Print the version of a toolchain component. - - Args: - executable: The toolchain component to check the version of. - versionFlag: The flag to pass to the executable to get the version. - """ - args = f'"{executable}" "{versionFlag}"' - try: - output = run(args, stdout=PIPE, shell=True).stdout.decode().strip() - match = re.search(regex, output, re.IGNORECASE) - return match.group(1) if match else "" - except Exception as e: - raise RuntimeError(f"Failed to get version when calling {args}: {e}") From f994830cc2afa6419fc0e68fc9a65deebe70168b Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 02:03:13 +0000 Subject: [PATCH 058/105] First pass at removing globals from SolutionStructs.py --- tensilelite/Tensile/BenchmarkProblems.py | 38 ++- tensilelite/Tensile/BenchmarkStructs.py | 4 +- tensilelite/Tensile/ClientWriter.py | 2 +- .../Tensile/Common/GlobalParameters.py | 5 +- tensilelite/Tensile/Common/Types.py | 53 ++++ tensilelite/Tensile/Contractions.py | 9 +- tensilelite/Tensile/LibraryIO.py | 84 ++++-- tensilelite/Tensile/SolutionLibrary.py | 8 +- tensilelite/Tensile/SolutionStructs.py | 269 ++++-------------- tensilelite/Tensile/Tensile.py | 50 +--- .../Tensile/TensileCreateLibrary/Run.py | 14 +- tensilelite/Tensile/Toolchain/Assembly.py | 12 +- tensilelite/Tensile/Toolchain/Component.py | 7 +- tensilelite/Tensile/Toolchain/Source.py | 5 +- 14 files changed, 233 insertions(+), 327 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index e7978e3a82..cfebcb5f81 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -43,12 +43,13 @@ from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain +from Tensile.Toolchain.Component import Assembler from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx -def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler, \ +def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ debugConfig: DebugConfig, isaInfoMap: Dict[str, IsaInfo]): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -61,7 +62,14 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxC solution.update(perm) # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, cxxCompiler, isaInfoMap) + solutionObject = Solution( + solution, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + assembler, + isaInfoMap + ) if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) @@ -75,14 +83,21 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, cxxC def _getCustomKernelSolutionObj( kernelName, internalSupportParams, - cxxCompiler: str, + assembler: Assembler, debugConfig: DebugConfig, \ isaInfoMap: Dict[str, IsaInfo], directory=CUSTOM_KERNEL_PATH ): """Creates the Solution object for a custom kernel""" config = getCustomKernelConfig(kernelName, internalSupportParams, directory) - return Solution(config, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, cxxCompiler, isaInfoMap) + return Solution( + config, + debugConfig.printIndexAssignmentInfo, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + assembler, + isaInfoMap + ) def _generateCustomKernelSolutions( @@ -90,7 +105,7 @@ def _generateCustomKernelSolutions( customKernels, internalSupportParams, failOnMismatch, - cxxCompiler: str, + assembler: Assembler, debugConfig: DebugConfig, isaInfoMap: Dict[str, IsaInfo] ): @@ -98,7 +113,7 @@ def _generateCustomKernelSolutions( solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler, debugConfig, isaInfoMap) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, assembler, debugConfig, isaInfoMap) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -208,6 +223,7 @@ def writeBenchmarkFiles( asmToolchain.assembler, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, isaInfoMap, ) newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) @@ -265,7 +281,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize print1("# Converting Config to BenchmarkProcess Object") print1(HR) print1("") - benchmarkProcess = BenchmarkProcess(problemTypeConfig, problemSizeGroupConfig) + benchmarkProcess = BenchmarkProcess(problemTypeConfig, problemSizeGroupConfig, debugConfig.printIndexAssignmentInfo) enableTileSelection = benchmarkProcess.problemType["TileAwareSelection"] groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) @@ -336,11 +352,11 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize maxPossibleSolutions = len(forkPermutations) regSolutions = _generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, str(srcToolchain.compiler.path), \ + benchmarkStep.constantParams, forkPermutations, asmToolchain.assembler, \ debugConfig, isaInfoMap) kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, str(srcToolchain.compiler.path), debugConfig, \ + not benchmarkStep.customKernelWildcard, asmToolchain.assembler, debugConfig, \ isaInfoMap) maxPossibleSolutions += len(kcSolutions) @@ -398,7 +414,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize solutions = None print1("# Using cached solution data") - ssProblemType = ProblemType(problemTypeConfig) + ssProblemType = ProblemType(problemTypeConfig, debugConfig.printIndexAssignmentInfo) conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) outFile = os.path.join(sourcePath, "ClientParameters.ini") @@ -467,7 +483,7 @@ def main( for idx, sizeGroupConfig in enumerate(problemSizeGroupConfigs): print2("ProblemTypeConfig: {}".format(problemTypeConfig)) - problemTypeObj = ProblemType(problemTypeConfig) + problemTypeObj = ProblemType(problemTypeConfig, debugConfig.printIndexAssignmentInfo) # using a suffix to check the csv version (for later addFromCSV()) csvSuffix = "_CSVWinner" if globalParameters["CSVExportWinner"] else "" diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index 6211083479..b6f5d84e23 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -92,9 +92,9 @@ def checkCDBufferAndStrides(problemType, problemSizes, isCEqualD): class BenchmarkProcess: """Representation of benchmarking parameters and resulting steps""" - def __init__(self, problemTypeConfig, problemSizeGroupConfig): + def __init__(self, problemTypeConfig, problemSizeGroupConfig, printIndexAssignmentInfo: bool): """Create from the two sections of a config for a BenchmarkProblem""" - self.problemType = ProblemType(problemTypeConfig) + self.problemType = ProblemType(problemTypeConfig, printIndexAssignmentInfo) self.isBatched = "Batched" in problemTypeConfig and problemTypeConfig["Batched"] print2("# BenchmarkProcess beginning {}".format(self.problemType)) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 739b612e73..05e971287c 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -30,7 +30,7 @@ from pathlib import Path from enum import Enum from glob import glob -from typing import Dict +from typing import List from . import ROOT_PATH from . import ClientExecutable diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 6cf25dfc7f..3abf0c2d7e 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -30,12 +30,11 @@ import time from collections import OrderedDict from copy import deepcopy -from typing import List, Dict +from typing import Dict from Tensile import __version__ from .Architectures import isaToGfx, SUPPORTED_ISA -from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps, initRegisterCaps from .Types import IsaVersion, IsaInfo from .Utilities import locateExe, versionIsCompatible, print1, print2, printExit, printWarning, \ verbosity @@ -222,7 +221,7 @@ ) globalParameters["PrintTensorBias"] = 0 # Print TensorBias after initialization globalParameters["PrintTensorAmaxD"] = 0 # Print AmaxD after validation -globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info +#globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info globalParameters["PrintWinnersOnly"] = False # Only print the solutions which become the fastest globalParameters["PrintCodeCommands"] = ( False # print the commands used to generate the code objects (asm,link,hip-clang, etc) diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index 875451fdb9..a0f8d3f573 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -51,3 +51,56 @@ class DebugConfig(NamedTuple): forceGenerateKernel: bool=False printSolutionRejectionReason: bool=False splitGSU: bool=False + printIndexAssignmentInfo: bool=False + + +def makeDebugConfig(config: dict) -> DebugConfig: + + enableAsserts = False + enableDebugA = False + enableDebugB = False + enableDebugC = False + expectedValueC = 16.0 + forceCExpectedValue = False + debugKernel = False + forceGenerateKernel = False + printSolutionRejectionReason = False + splitGSU = False + printIndexAssignmentInfo = False + + if "EnableAsserts" in config: + enableAsserts = config["EnableAsserts"] + if "EnableDebugA" in config: + enableDebugA = config["EnableDebugA"] + if "EnableDebugB" in config: + enableDebugB = config["EnableDebugB"] + if "EnableDebugC" in config: + enableDebugC = config["EnableDebugC"] + if "ExpectedValueC" in config: + expectedValueC = config["ExpectedValueC"] + if "ForceCExpectedValue" in config: + forceCExpectedValue = config["ForceCExpectedValue"] + if "DebugKernel" in config: + debugKernel = config["DebugKernel"] + if "ForceGenerateKernel" in config: + forceGenerateKernel = config["ForceGenerateKernel"] + if "PrintSolutionRejectionReason" in config: + printSolutionRejectionReason = config["PrintSolutionRejectionReason"] + if "SplitGSU" in config: + splitGSU = config["SplitGSU"] + if "PrintIndexAssignmentInfo" in config: + printIndexAssignmentInfo = config["PrintIndexAssignmentInfo"] + + return DebugConfig( + enableAsserts, + enableDebugA, + enableDebugB, + enableDebugC, + expectedValueC, + forceCExpectedValue, + debugKernel, + forceGenerateKernel, + printSolutionRejectionReason, + splitGSU, + printIndexAssignmentInfo, + ) diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 551a3d2f30..39de4f948a 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -32,6 +32,7 @@ from .SolutionStructs import Solution as OriginalSolution from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering, \ IsaVersion, IsaInfo +from Tensile.Toolchain.Component import Assembler @state_key_ordering class FreeIndex: @@ -659,11 +660,11 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, cxxCompiler: str, isaInfoMap: Dict[str, IsaInfo]): - return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, solution.srcName) + def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, assembler: Assembler, isaInfoMap: Dict[str, IsaInfo]): + return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, solution.srcName) @classmethod - def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, cxxCompiler, isaInfoMap, srcName = "", deviceInfo=None): + def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, assembler, isaInfoMap, srcName = "", deviceInfo=None): rv = cls() if 'SolutionNameMin' in d: @@ -707,7 +708,7 @@ def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcName) + rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, srcName) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index b18361411f..4995786d55 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -182,24 +182,50 @@ def read(filename, customizedLoader=False): else: printExit("Unrecognized read format {}".format(extension)) + def readYAML(filename): """Reads and returns YAML data from file.""" with open(filename, "r") as f: data = yaml.load(f, yamlLoader) return data + def readJson(filename): """Reads and returns JSON data from file.""" with open(filename, "r") as f: data = json.loads(f.read()) return data -def parseSolutionsFile(filename, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap): - """Wrapper function to read and parse a solutions file.""" - return parseSolutionsData(read(filename), filename, cxxCompiler, splitGSU, printSolutionRejectionReason, isaInfoMap) - -def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap): +def parseSolutionsFile( + filename, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + isaInfoMap + ): + """Wrapper function to read and parse a solutions file.""" + return parseSolutionsData( + read(filename), + filename, + assembler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + isaInfoMap + ) + + +def parseSolutionsData( + data, + srcFile, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + isaInfoMap + ): """Parses problem sizes and solutions from the data of a solutions file.""" if len(data) < 3: printExit("Solution file {} is missing required fields (len = {} < 3" \ @@ -226,7 +252,15 @@ def parseSolutionsData(data, srcFile, cxxCompiler, splitGSU: bool, printSolution # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcFile) + solutionObject = Solution( + solutionState, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + assembler, + isaInfoMap, + srcFile + ) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) @@ -244,33 +278,36 @@ class LibraryLogic(NamedTuple): def parseLibraryLogicFile( filename, - cxxCompiler, + assembler, splitGSU: bool, printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool ): """Wrapper function to read and parse a library logic file.""" return parseLibraryLogicData( - read(filename, True), - filename, - cxxCompiler, + read(filename, True), + filename, + assembler, splitGSU, - printSolutionRejectionReason, - archs, + printSolutionRejectionReason, + printIndexAssignmentInfo, + archs, isaInfoMap, lazyLibraryLoading ) def parseLibraryLogicData( - data, - srcFile, - cxxCompiler, - splitGSU: bool, - printSolutionRejectionReason: bool, - archs, + data, + srcFile, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool ): @@ -298,10 +335,10 @@ def parseLibraryLogicData( .format(srcFile, data["MinimumRequiredVersion"], __version__) ) # unpack problemType - problemType = ProblemType(data["ProblemType"]) + problemType = ProblemType(data["ProblemType"], printIndexAssignmentInfo) # unpack solution - def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: + def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: if solutionState["KernelLanguage"] == "Assembly": solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) solutionState["CUCount"] = data["CUCount"] @@ -319,7 +356,7 @@ def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap, srcFile) + solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, srcFile) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -331,14 +368,15 @@ def solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) -> Solution: printExit(f"ProblemType in library logic file {srcFile} doesn't match solution(idx={solIdx}): \n{results}") return solutionObject - solutions = [solutionStateToSolution(solutionState, cxxCompiler, isaInfoMap) for solutionState in data["Solutions"]] + solutions = [solutionStateToSolution(solutionState, assembler, isaInfoMap) for solutionState in data["Solutions"]] newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState( data, solutions, splitGSU, printSolutionRejectionReason, - cxxCompiler, + printIndexAssignmentInfo, + assembler, isaInfoMap, lazyLibraryLoading ) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 445016f863..ab194ec577 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -302,6 +302,7 @@ def FromOriginalState(cls, origSolutions, splitGSU: bool, printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, cxxCompiler, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool, @@ -466,6 +467,7 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions, splitGSU, printSolutionRejectionReason, + printIndexAssignmentInfo, cxxCompiler, isaInfoMap, lazyLibraryLoading, @@ -476,7 +478,7 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, cxxCompiler, isaInfoMap) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -497,8 +499,8 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, cxxCompiler, isaInfoMap) for s in solutions]) + def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap): + solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, cxxCompiler, isaInfoMap) for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index f6c0a4f9eb..cb6d51b93b 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -44,7 +44,8 @@ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ validGEMMTypes, HPATypes, roundUp, validWMMA, \ - INDEX_CHARS, IsaVersion + INDEX_CHARS, IsaVersion, SemanticVersion +from Tensile.Toolchain.Component import Assembler from collections import OrderedDict from collections.abc import Mapping @@ -98,7 +99,7 @@ class Fbs(Enum): # name of solution should begin with name of problemType, and arguments can be listed out explicitly class ProblemType(Mapping): ######################################## - def __init__(self, config): + def __init__(self, config, printIndexAssignmentInfo: bool): self.state = {} for key in defaultProblemType: @@ -182,7 +183,7 @@ def __init__(self, config): printExit("Unsupported OperationType = %s" % self["OperationType"]) self.state["AssignedDerivedParameters"] = False - ProblemType.assignDerivedParameters(self.state) + ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) for tc in ('A', 'B'): for sc in self["SetConstStride%s"%tc] : @@ -332,7 +333,7 @@ def isGEMM(self): ######################################## # determine d0, d1, dU @staticmethod - def assignDerivedParameters(state): + def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): if "AssignedDerivedParameters" in state: if state["AssignedDerivedParameters"]: return @@ -366,7 +367,7 @@ def assignDerivedParameters(state): else: printExit("invalid index %u (expected summation but not (inA and inB))" % i) # print index assignments - if globalParameters["PrintIndexAssignments"]: + if printIndexAssignmentInfo: print("IndicesFree: %s" % state["IndicesFree"]) print("IndicesBatch: %s" % state["IndicesBatch"]) print("IndicesSum: %s" % state["IndicesSummation"]) @@ -436,7 +437,7 @@ def assignDerivedParameters(state): state["TLUB"] = strideIdxB < unrollIdxB #state["TLUB"] = True # hack - if globalParameters["PrintIndexAssignments"]: + if printIndexAssignmentInfo: print("TLUA: %s (stridePosA(%d) 0) - dbFract = 0 - - # parDim, perpDim define the LoadTile and are measured in elements - if state["ProblemType"]["TLU%s"%tc]: - parDim = state["MacroTile%s"%tc] - perpDim = depthU - else: - parDim = depthU - perpDim = state["MacroTile%s"%tc] - - if dbFract: - print("\ninfo: %s Fractional MT%u_%u_%u Par=%u Perp=%u WG%02u_%02u_%02u NumThreads=%u GRWV%s=%u" \ - % (tc, state["MacroTile0"], state["MacroTile1"], depthU, \ - parDim, perpDim, \ - state["WorkGroup"][0], state["WorkGroup"][1], state["LocalSplitU"], \ - state["NumThreads"], tc, state["GlobalReadVectorWidth%s"%tc])) - - # Try to find a GRVW which is smaller than the LSC and also does not force - # the LSC to wrap - both of these conditions can be tested with lsc % grvw ==0. - # Each iteration divides GRWV by 2 which provides finer granularity - # and a possible opportunity to handle the lsc - grvw = state["GlobalReadVectorWidth%s"%tc] - minGrvw = 2 if state["ProblemType"]["DataType"].isHalf() and \ - globalParameters["ArchCaps"][state["ISA"]]["HasEccHalf"] else 1 # why were we using current ISA here? - bestVw = -1 - while grvw >= minGrvw: - # Per instruction across the entire group: - elementsLoadedPerInst = state["NumThreads"]*grvw - mik = 1 - if (state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]): - mik = state["MatrixInstK"] * state["LocalSplitU"] // state["MIInputPerThread"] - elementsLoadedPerInst //= mik - # LSC, LSP - #elements loaded along specified dim with each load - if parDim >= elementsLoadedPerInst: - # entire work-group can work on (part) of the same row - state["LSC%s"%tc] = elementsLoadedPerInst - state["LSP%s"%tc] = mik if state["ProblemType"]["TLU%s"%tc] else state["MatrixInstK"] - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim , state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = 1 - else: - # work-group exceeds read dimension so wraps to multiple rows - state["LSC%s"%tc] = parDim - state["LSP%s"%tc] = min(perpDim, elementsLoadedPerInst // parDim) - state["NumLoadsCoalesced%s"%tc] = 1 - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - # Vector loads can't wrap to next P dim, so LSC must be divisible by vector elements; - if dbFract: - print(" lsc search : lsc(%u) %% grvw(%u) = %u (?0)" % (state["LSC%s"%tc], grvw, state["LSC%s"%tc] % grvw)) - if state["LSC%s"%tc] % grvw == 0: - bestVw = grvw - # Try to shrink GRVW if possible while keeping same LSC and LSP: - # For example, avoid cases where we use a GRVW=4 with many empty addresses - # when a GRVW=1 will do instead. - validElementsLoadedPerInst = state["LSC%s"%tc] * state["LSP%s"%tc] - grvw //= 2 - while grvw >= minGrvw: - elementsLoadedPerInst = state["NumThreads"]*grvw//mik - if elementsLoadedPerInst < validElementsLoadedPerInst: - break # Went too far, not enough load elements at this VW - if state["LSC%s"%tc] % grvw == 0: - if dbFract: - print(" stepdown success (valid)elementsLoadedPerInst=", validElementsLoadedPerInst, "/", elementsLoadedPerInst, "grvw=", grvw, "lsc=", state["LSC%s"%tc]) - bestVw = grvw - grvw //= 2 - break - - # TODO - could have this generate dwordx3 loads in addition, step down by 1 instead of div2 - # Would need to change asm code gen to generate x3 - grvw //= 2 - # end-- while loop - - if bestVw == -1: - if dbFract: - print ("reject fractional - no acceptable tile dim? GlobalReadVectorWidth%s"%tc, \ - state["GlobalReadVectorWidth%s"%tc]) - return False # could not find a solution, perhaps only possible for half ? - - state["GlobalReadVectorWidth%s"%tc] = bestVw - if bestVw != state["GlobalReadVectorWidth%s"%tc]: - if dbFract: - print(" reducing GlobalReadVectorWidth%s from %u to %u" \ - % (tc, state["GlobalReadVectorWidth%s"%tc], bestVw)) - - # How many loads per threads in each dimension. - # threads which are outside the global read tile bounds will be clipped - # in the assembly code generator. - # Multiply the LSC*GRVW - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim, state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - nlc = state["NumLoadsCoalesced%s"%tc] - nlp = state["NumLoadsPerpendicular%s"%tc] - - # LoadTile must at least cover the MacroTile: - assert(nlc*state["LSC%s"%tc] >= parDim) - assert(nlp*state["LSP%s"%tc] >= perpDim) - - perpOverhang = perpDim % state["LSP%s"%tc] - state["fractionalPerpOverhang%s"%tc] = perpOverhang - if dbFract: - # how many threads compute Global Read Offsets (GRO) that are not used - print(" PerLoadTile=%ux%u elements Loads/WI=%ux%u LoadTile/WI=%ux%u (MT=%ux%u), %u/%u = %.1f%% WI GRO used %s" \ - % (state["LSC%s"%tc], state["LSP%s"%tc], \ - nlc, nlp, \ - nlc*state["LSC%s"%tc], nlp*state["LSP%s"%tc], \ - parDim, perpDim, \ - parDim*perpDim, \ - nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc], \ - float(parDim*perpDim), \ - float(nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc]) * 100.0) \ - ) - - for p in range(0,nlp): - elementWidth = 4 - if p != nlp-1: - perp = state["LSP%s"%tc] - else: - perp = perpOverhang if perpOverhang else state["LSP%s"%tc] - - validElements = state["LSC%s"%tc] * perp - print(" buffer_load_element_x%u %ux%ux%u bytes, %u/%u valid GRO" %\ - (state["GlobalReadVectorWidth%s"%tc], \ - state["LSC%s"%tc], perp, \ - elementWidth, \ - validElements//state["GlobalReadVectorWidth%s"%tc], - state["NumThreads"])) - - return True - - @staticmethod def MatrixInstructionToMIParameters(state, printRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): isa = state["ISA"] @@ -2122,7 +1947,15 @@ def getDivisorName(state, tC): ######################################## # assign all derived parameters @staticmethod - def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool, isaInfoMap): + def assignDerivedParameters( + state, + splitGSU: bool, + printRejectionReason: bool, + printIndexAssignmentInfo: bool, + isaInfoMap, + rocmVersion: SemanticVersion, + #depthUParams: Dict + ): state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. if "F32XdlMathOp" in state["ProblemType"] \ @@ -2221,7 +2054,7 @@ def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool, i if state["VectorStore"] == -1: state["_VectorStore"] = 1 # default, may be changed if needed to generate a valid kernel - ProblemType.assignDerivedParameters(state["ProblemType"]) + ProblemType.assignDerivedParameters(state["ProblemType"], printIndexAssignmentInfo) if not state["Valid"]: print2("in assignDerivedParameters, state['Valid'] = False") return @@ -2559,7 +2392,18 @@ def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool, i state["ValidDepthU"] = True state["DepthU"] = depthuList[index[0]] Solution.depthUIteration( - state, index, depthuList, problemType, isa, bufferLoad, packedC0, packedC1, printRejectionReason, isaInfoMap + state, + index, + depthuList, + problemType, + isa, + bufferLoad, + packedC0, + packedC1, + printRejectionReason, + isaInfoMap, + rocmVersion, + #depthUParams ) if state["Valid"] or (state["ValidDepthU"] and (not state["Valid"])): break @@ -2568,7 +2412,7 @@ def assignDerivedParameters(state, splitGSU: bool, printRejectionReason: bool, i break if "ValidDepthU" in state: del state["ValidDepthU"] - + def depthUIteration( state, index, @@ -2579,7 +2423,9 @@ def depthUIteration( packedC0, packedC1, printRejectionReason: bool, - isaInfoMap: Dict[str, IsaInfo] + isaInfoMap: Dict[IsaVersion, IsaInfo], + rocmVersion: SemanticVersion, + #depthUParams: NamedTuple ): ######################################## # Auto search for DepthU starts here @@ -4231,10 +4077,7 @@ def calcEpilogueTurns(factorDims: List) -> int: #Need to force disabling PreloadKernArgs if compiler does not support #Can not just reject the solution since the user library may find any solutions if state["PreloadKernArgs"]: - hipccver = globalParameters['HipClangVersion'].split(".") - hipccMaj = int(hipccver[0]) - hipccPatch = int(hipccver[2].split("-")[0]) - if not (hipccMaj >= 6 and hipccPatch >= 32650 and (isa == (9, 0, 10) or isa[:2] == (9, 4))): + if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650 and (isa == (9, 0, 10) or isa[:2] == (9, 4))): #print("Force to Disable PreloadKernArgs since this hipcc version doesn't support",) state["PreloadKernArgs"] = 0 diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 07522ce3bb..066b37be6d 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -37,7 +37,7 @@ from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ - detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap, isaToGfx + detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap, isaToGfx, makeDebugConfig from Tensile.KernelWriter import DebugConfig from Tensile.Toolchain.Assembly import AssemblyToolchain, makeAssemblyToolchain from Tensile.Toolchain.Source import SourceToolchain, makeSourceToolchain @@ -329,54 +329,6 @@ def store_max_frequency(max_frequency): return False -def makeDebugConfig(config: dict) -> DebugConfig: - - enableAsserts = False - enableDebugA = False - enableDebugB = False - enableDebugC = False - expectedValueC = 16.0 - forceCExpectedValue = False - debugKernel = False - forceGenerateKernel = False - printSolutionRejectionReason = False - splitGSU = False - - if "EnableAsserts" in config: - enableAsserts = config["EnableAsserts"] - if "EnableDebugA" in config: - enableDebugA = config["EnableDebugA"] - if "EnableDebugB" in config: - enableDebugB = config["EnableDebugB"] - if "EnableDebugC" in config: - enableDebugC = config["EnableDebugC"] - if "ExpectedValueC" in config: - expectedValueC = config["ExpectedValueC"] - if "ForceCExpectedValue" in config: - forceCExpectedValue = config["ForceCExpectedValue"] - if "DebugKernel" in config: - debugKernel = config["DebugKernel"] - if "ForceGenerateKernel" in config: - forceGenerateKernel = config["ForceGenerateKernel"] - if "PrintSolutionRejectionReason" in config: - printSolutionRejectionReason = config["PrintSolutionRejectionReason"] - if "SplitGSU" in config: - splitGSU = config["SplitGSU"] - if "SaveTemps" in config: - splitGSU = config["SaveTemps"] - - return DebugConfig(enableAsserts, - enableDebugA, - enableDebugB, - enableDebugC, - expectedValueC, - forceCExpectedValue, - debugKernel, - forceGenerateKernel, - printSolutionRejectionReason, - splitGSU) - - ################################################################################ # Tensile # - below entry points call here diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index bbab2e8017..6f109230b1 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -29,7 +29,7 @@ import shutil from pathlib import Path from timeit import default_timer as timer -from typing import Dict, List, NamedTuple, Optional, Sequence, Union +from typing import List, NamedTuple, Optional, Sequence, Union from Tensile import SOURCE_PATH, LibraryIO from Tensile.Common import ( @@ -43,6 +43,7 @@ ParallelMap2, SemanticVersion, architectureMap, + globalParameters, assignGlobalParameters, ensurePath, isaToGfx, @@ -268,6 +269,7 @@ def assemble(ret): codeObjectFiles += buildAssemblyCodeObjectFiles( asmToolchain.linker, asmToolchain.bundler, + globalParameters["ROCmLdPath"], asmKernels, kernelWriterAssembly, destLibPath, @@ -341,10 +343,10 @@ def assemble(ret): multiArg=False, return_as="list" ) - buildAssemblyCodeObjectFiles( asmToolchain.linker, asmToolchain.bundler, + globalParameters["ROCmLdPath"], asmKernels, kernelWriterAssembly, destLibPath, @@ -435,7 +437,7 @@ def generateKernelObjectsFromSolutions(solutions): @timing -def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler, isaInfoMap): +def generateLogicDataAndSolutions(logicFiles, args, assembler: Assembler, isaInfoMap): if ";" in args["Architecture"]: archs = args["Architecture"].split(";") # user arg list format @@ -447,12 +449,14 @@ def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler, isaInfoMap): nextSolIndex = 0 splitGSU = False printSolutionRejectionReason = False + printIndexAssignmentInfo = False fIter = zip( logicFiles, - itertools.repeat(cxxCompiler), + itertools.repeat(assembler), itertools.repeat(splitGSU), itertools.repeat(printSolutionRejectionReason), + itertools.repeat(printIndexAssignmentInfo), itertools.repeat(archs), itertools.repeat(isaInfoMap), itertools.repeat(args["LazyLibraryLoading"]), @@ -628,7 +632,7 @@ def validLogicFile(p: Path): print2("# %s" % logicFile) solutions, masterLibraries = generateLogicDataAndSolutions( - logicFiles, arguments, cxxCompiler, isaInfoMap + logicFiles, arguments, asmToolchain.assembler, isaInfoMap ) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 12176af594..04b14c1a4a 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import List, Union, NamedTuple -from ..Common import globalParameters, print2, ensurePath, isaToGfx +from ..Common import print2, ensurePath, isaToGfx from ..KernelWriterAssembly import KernelWriterAssembly from ..SolutionStructs import Solution @@ -49,7 +49,7 @@ def makeAssemblyToolchain(assembler_path, bundler_path, co_version, build_id_kin return AssemblyToolchain(compiler, linker, bundler) -def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: +def _batchObjectFiles(ldPath: str, objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: numObjFiles = len(objFiles) if numObjFiles <= maxObjFiles: @@ -63,7 +63,7 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF for batch, filename in zip(batchedObjFiles, newObjFiles): if len(batch) > 1: - args = [globalParameters["ROCmLdPath"], "-r"] + batch + [ "-o", filename] + args = [ldPath, "-r"] + batch + [ "-o", filename] print2(f"Linking object files into fewer object files: {' '.join(args)}") subprocess.check_call(args) newObjFilesOutput.append(filename) @@ -76,6 +76,7 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF def buildAssemblyCodeObjectFiles( linker: Linker, bundler: Bundler, + ldPath: str, kernels: List[Solution], writer: KernelWriterAssembly, destDir: Union[Path, str], @@ -100,9 +101,6 @@ def buildAssemblyCodeObjectFiles( extCo = ".co" extCoRaw = ".co.raw" - destDir = Path(ensurePath(destDir)) - asmDir = Path(ensurePath(asmDir)) - archKernelMap = collections.defaultdict(list) for k in filter(isAsm, kernels): archKernelMap[tuple(k['ISA'])].append(k) @@ -123,7 +121,7 @@ def buildAssemblyCodeObjectFiles( if coName: coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(useShortNames, kernel) + extObj))) for coFileRaw, objFiles in coFileMap.items(): - objFiles = _batchObjectFiles(objFiles, coFileRaw) + objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) linker(objFiles, str(coFileRaw)) coFile = destDir / coFileRaw.name.replace(extCoRaw, extCo) if compress: diff --git a/tensilelite/Tensile/Toolchain/Component.py b/tensilelite/Tensile/Toolchain/Component.py index 06a6c653c0..6d5e40edbd 100644 --- a/tensilelite/Tensile/Toolchain/Component.py +++ b/tensilelite/Tensile/Toolchain/Component.py @@ -89,7 +89,7 @@ def version(self): @property def rocm_version(self): - return Component.rocm_version + return Component._rocm_version class Assembler(Component): @@ -122,7 +122,7 @@ def __init__(self, component_path: Path, co_version: str, debug: bool=False): """ super(Assembler, self).__init__(component_path) - + self._code_object_version = co_version self._default_args = [ *split(environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')), @@ -154,6 +154,9 @@ def __call__(self, targetGfx: str, wavefrontSize: int, srcPath: str, destPath: s ] return _invoke(args, "Assembling assembly source code into object file (.s -> .o)") + @property + def code_object_version(self): + return self._code_object_version class Compiler(Component): """ diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index b572be3e1b..3cd54c5b35 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -30,7 +30,7 @@ from timeit import default_timer as timer from typing import List, Union, NamedTuple -from ..Common import globalParameters, print1, ensurePath +from ..Common import print1, ensurePath from .Component import Compiler, Bundler @@ -98,9 +98,6 @@ def buildSourceCodeObjectFiles( destDir = Path(ensurePath(destDir)) kernelPath = Path(kernelPath) - if "CmakeCxxCompiler" in globalParameters and globalParameters["CmakeCxxCompiler"] is not None: - os.environ["CMAKE_CXX_COMPILER"] = globalParameters["CmakeCxxCompiler"] - objFilename = kernelPath.stem + '.o' coPathsRaw = [] coPaths= [] From 4abe9981e220055c7714e2f0c763c72cb58da0ca Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 03:17:55 +0000 Subject: [PATCH 059/105] Fix broken tests --- tensilelite/Tensile/LibraryLogic.py | 8 ++++---- tensilelite/Tensile/Tensile.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index bea35f2063..8047cb7169 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -1431,7 +1431,7 @@ def indicesToSerial(self, solutionIdx, indices ): return serial -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): +def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap: Dict[str, IsaInfo]): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1474,7 +1474,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU, printSolutionRejectionReason, isaInfoMap) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, isaInfoMap) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1547,7 +1547,7 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool, printSolutionRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): +def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap: Dict[str, IsaInfo]): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU, printSolutionRejectionReason, isaInfoMap) + generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, isaInfoMap) diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 066b37be6d..9c4284608c 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -125,6 +125,7 @@ def executeStepsInConfig( outputPath, debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, isaInfoMap, ) print1("") From b99eebe877882633625aad8805113574292873b2 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 04:41:13 +0000 Subject: [PATCH 060/105] Completely remove globalParams from solutionstructs --- tensilelite/Tensile/BenchmarkProblems.py | 37 ++++++++----- .../Tensile/Common/GlobalParameters.py | 5 +- tensilelite/Tensile/Common/Types.py | 15 ++++++ tensilelite/Tensile/Contractions.py | 53 ++++++++++++++++--- tensilelite/Tensile/LibraryIO.py | 33 +++++++++--- tensilelite/Tensile/LibraryLogic.py | 47 ++++++++++++++-- tensilelite/Tensile/SolutionLibrary.py | 39 +++++++++++--- tensilelite/Tensile/SolutionStructs.py | 32 +++++------ tensilelite/Tensile/Tensile.py | 12 +++-- .../Tensile/TensileCreateLibrary/Run.py | 2 + 10 files changed, 213 insertions(+), 62 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index cfebcb5f81..2118f56bbb 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -46,11 +46,12 @@ from Tensile.Toolchain.Component import Assembler from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ - BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx, \ + DepthUConfig def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ - debugConfig: DebugConfig, isaInfoMap: Dict[str, IsaInfo]): + debugConfig: DebugConfig, depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo]): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") @@ -67,6 +68,7 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, + depthUConfig, assembler, isaInfoMap ) @@ -83,8 +85,9 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse def _getCustomKernelSolutionObj( kernelName, internalSupportParams, - assembler: Assembler, - debugConfig: DebugConfig, \ + assembler: Assembler, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo], directory=CUSTOM_KERNEL_PATH ): @@ -95,18 +98,19 @@ def _getCustomKernelSolutionObj( debugConfig.printIndexAssignmentInfo, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, + depthUConfig, assembler, isaInfoMap ) def _generateCustomKernelSolutions( - problemType, - customKernels, - internalSupportParams, + problemType, + customKernels, + internalSupportParams, failOnMismatch, - assembler: Assembler, - debugConfig: DebugConfig, + assembler: Assembler, + debugConfig: DebugConfig, isaInfoMap: Dict[str, IsaInfo] ): """Creates a list with a Solution object for each name in customKernel""" @@ -159,6 +163,7 @@ def writeBenchmarkFiles( sourcePath: Path, useShortNames: bool, debugConfig: DebugConfig, + depthUConfig: DepthUConfig, deviceId: int, isaInfoMap: Dict[str, IsaInfo] ): @@ -224,6 +229,7 @@ def writeBenchmarkFiles( debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, + depthUConfig, isaInfoMap, ) newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) @@ -270,8 +276,9 @@ def writeBenchmarkFiles( def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, - debugConfig: DebugConfig, deviceId: int, isaInfoMap: Dict[str, IsaInfo] + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, + debugConfig: DebugConfig, depthUConfig: DepthUConfig, deviceId: int, + isaInfoMap: Dict[str, IsaInfo] ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -353,7 +360,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize regSolutions = _generateForkedSolutions(benchmarkProcess.problemType, \ benchmarkStep.constantParams, forkPermutations, asmToolchain.assembler, \ - debugConfig, isaInfoMap) + debugConfig, depthUConfig, isaInfoMap) kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ not benchmarkStep.customKernelWildcard, asmToolchain.assembler, debugConfig, \ @@ -383,10 +390,10 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize # write benchmarkFiles prevCount = len(solutions) codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ - benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ + benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath, useShortNames, debugConfig, deviceId, isaInfoMap) + sourcePath, useShortNames, debugConfig, depthUConfig, deviceId, isaInfoMap) # ^ this mutates solutions # write cache data @@ -461,6 +468,7 @@ def main( buildTmpPath: Path, useShortNames: bool, debugConfig: DebugConfig, + depthUConfig: DepthUConfig, deviceId: int, isaInfoMap: Dict[str, IsaInfo] ): @@ -514,6 +522,7 @@ def main( benchmarkProblemsPath, useShortNames, debugConfig, + depthUConfig, deviceId, isaInfoMap ) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 3abf0c2d7e..89a4b35189 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -237,9 +237,6 @@ globalParameters["Platform"] = 0 # select opencl platform # shouldn't need to change -globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy -globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use -globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. globalParameters["ClientExecutionLockPath"] = ( None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. ) @@ -272,7 +269,7 @@ # limitation for training globalParameters["MaxWorkspaceSize"] = 128 * 1024 * 1024 # max workspace for training (128MB) -globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) +#globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) # control if a solution is run for a given problem globalParameters["GranularityThreshold"] = 0.0 diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index a0f8d3f573..f36e0777a5 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -40,6 +40,21 @@ class SemanticVersion(NamedTuple): IsaVersion = SemanticVersion + +class DepthUConfig(NamedTuple): + deviceLDS: int=65536 + maxLDS: int=65536 + + +def makeDepthUConfig(config: dict) -> DepthUConfig: + deviceLDS = maxLDS = 65536 + if "DeviceLDS" in config: + deviceLDS = config["DeviceLDS"] + if "MaxLDS" in config: + maxLDS = config["MaxLDS"] + return DepthUConfig(deviceLDS, maxLDS) + + class DebugConfig(NamedTuple): enableAsserts: bool=False enableDebugA: bool=False diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 39de4f948a..2db3e302ef 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -30,8 +30,8 @@ from . import Properties from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution -from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering, \ - IsaVersion, IsaInfo +from .Common import gfxToIsa, internalParameters, state, state_key_ordering, \ + IsaInfo, DepthUConfig from Tensile.Toolchain.Component import Assembler @state_key_ordering @@ -515,7 +515,7 @@ def CompoundPredicates(cls, state, problemType): rv += [cls('BufferStoreOffsetLimitCheck', value=state['MacroTile1'])] if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None and not state["StreamK"]: - value = globalParameters['MinKForGSU'] + value = 32 # globalParameters['MinKForGSU'] rv += [cls('GlobalSplitUCheckMinK', value=[value, state["GlobalSplitU"]])] if ('WorkGroupMappingXCC' in state) and ('WorkGroupMappingXCCGroup' in state): @@ -660,11 +660,41 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, assembler: Assembler, isaInfoMap: Dict[str, IsaInfo]): - return cls.FromOriginalState(solution._state, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, solution.srcName) + def FromSolutionStruct( + cls, + solution, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + assembler: Assembler, + isaInfoMap: Dict[str, IsaInfo] + ): + return cls.FromOriginalState( + solution._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + solution.srcName + ) @classmethod - def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, assembler, isaInfoMap, srcName = "", deviceInfo=None): + def FromOriginalState( + cls, + d, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + #mink + assembler, + isaInfoMap, + srcName = "", + deviceInfo=None + ): rv = cls() if 'SolutionNameMin' in d: @@ -708,7 +738,16 @@ def FromOriginalState(cls, d, splitGSU: bool, printSolutionRejectionReason: bool d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, srcName) + rv.originalSolution = OriginalSolution( + d, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + srcName + ) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 4995786d55..a87a188568 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -26,7 +26,9 @@ from .SolutionStructs import Solution, ProblemSizes, ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__, print1, IsaInfo +from .Common import gfxToIsa, printExit, printWarning, print2, \ + versionIsCompatible, __version__, \ + IsaInfo, DepthUConfig from typing import NamedTuple, List, Dict import os @@ -203,6 +205,7 @@ def parseSolutionsFile( splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, isaInfoMap ): """Wrapper function to read and parse a solutions file.""" @@ -213,6 +216,7 @@ def parseSolutionsFile( splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, + depthUConfig, isaInfoMap ) @@ -224,6 +228,7 @@ def parseSolutionsData( splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, isaInfoMap ): """Parses problem sizes and solutions from the data of a solutions file.""" @@ -257,6 +262,7 @@ def parseSolutionsData( splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, + depthUConfig, assembler, isaInfoMap, srcFile @@ -277,11 +283,12 @@ class LibraryLogic(NamedTuple): library: SolutionLibrary.MasterSolutionLibrary def parseLibraryLogicFile( - filename, - assembler, - splitGSU: bool, - printSolutionRejectionReason: bool, - printIndexAssignmentInfo: bool, + filename, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool @@ -294,6 +301,7 @@ def parseLibraryLogicFile( splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, + depthUConfig, archs, isaInfoMap, lazyLibraryLoading @@ -307,6 +315,7 @@ def parseLibraryLogicData( splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool @@ -356,7 +365,16 @@ def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, assembler, isaInfoMap, srcFile) + solutionObject = Solution( + solutionState, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + srcFile + ) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -376,6 +394,7 @@ def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, + depthUConfig, assembler, isaInfoMap, lazyLibraryLoading diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 8047cb7169..dc5bcc1427 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -26,7 +26,7 @@ from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -1431,7 +1431,17 @@ def indicesToSerial(self, solutionIdx, indices ): return serial -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap: Dict[str, IsaInfo]): +def generateLogic( + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler: str, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo] + ): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1474,7 +1484,15 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str, printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, isaInfoMap) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile( + solutionsFileName, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap + ) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1547,7 +1565,26 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap: Dict[str, IsaInfo]): +def main( + config, + cxxCompiler: str, + outputPath: Path, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo] + ): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, isaInfoMap) + generateLogic( + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap + ) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index ab194ec577..c97e02091d 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -29,7 +29,7 @@ from . import Hardware from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state, IsaInfo, gfxToIsa +from .Common import state, IsaInfo, gfxToIsa, DepthUConfig class SingleSolutionLibrary: Tag = "Single" @@ -303,7 +303,8 @@ def FromOriginalState(cls, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, - cxxCompiler, + depthUConfig: DepthUConfig, + assembler, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool, solutionClass=Contractions.Solution, @@ -468,7 +469,8 @@ def selection(d, problemType, solutions, library, placeholderName): splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, - cxxCompiler, + depthUConfig, + assembler, isaInfoMap, lazyLibraryLoading, solutionClass, @@ -478,7 +480,15 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, cxxCompiler, isaInfoMap) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct( + s, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap + ) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -499,8 +509,25 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler, splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, isaInfoMap): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, splitGSU, printSolutionRejectionReason, printIndexAssignmentInfo, cxxCompiler, isaInfoMap) for s in solutions]) + def BenchmarkingLibrary( + cls, + solutions, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap + ): + solutionObjs = list([Contractions.Solution.FromOriginalState( + s._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap) + for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index cb6d51b93b..9ed3505462 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -40,11 +40,12 @@ from .Common import assignParameterWithDefault, IsaInfo, \ defaultProblemType, defaultSolution, \ defaultInternalSupportParams, \ - globalParameters, internalParameters, \ + internalParameters, \ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ validGEMMTypes, HPATypes, roundUp, validWMMA, \ - INDEX_CHARS, IsaVersion, SemanticVersion + INDEX_CHARS, IsaVersion, SemanticVersion, \ + DepthUConfig from Tensile.Toolchain.Component import Assembler from collections import OrderedDict @@ -1060,6 +1061,7 @@ def __init__( splitGSU: bool, printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, assembler: Assembler, isaInfoMap: Dict[IsaVersion, IsaInfo], srcName: str = "" @@ -1122,7 +1124,7 @@ def __init__( printIndexAssignmentInfo, isaInfoMap, assembler.rocm_version, - #depthUParams, + depthUConfig, ) self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None @@ -1954,7 +1956,7 @@ def assignDerivedParameters( printIndexAssignmentInfo: bool, isaInfoMap, rocmVersion: SemanticVersion, - #depthUParams: Dict + depthUConfig: DepthUConfig ): state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. #enable F32 xDL MathOp only when the input type is f32. @@ -2403,7 +2405,7 @@ def assignDerivedParameters( printRejectionReason, isaInfoMap, rocmVersion, - #depthUParams + depthUConfig, ) if state["Valid"] or (state["ValidDepthU"] and (not state["Valid"])): break @@ -2425,7 +2427,7 @@ def depthUIteration( printRejectionReason: bool, isaInfoMap: Dict[IsaVersion, IsaInfo], rocmVersion: SemanticVersion, - #depthUParams: NamedTuple + depthUConfig: DepthUConfig ): ######################################## # Auto search for DepthU starts here @@ -2694,7 +2696,7 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo padA, padB, padM = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) ldsBlockSizePerPadA, ldsBlockSizePerPadB = calcLdsBlockSizePerPad(state["LocalReadVectorWidth"]) ldsNumBytesA, ldsNumBytesAlignedA, ldsNumBytesB, ldsNumBytesAlignedB, ldsNumBytesMetadata, ldsNumBytesAlignedMetadata = calcLdsNumBytes(padA, ldsBlockSizePerPadA, padB, ldsBlockSizePerPadB) - if (ldsNumBytesAlignedA + ldsNumBytesAlignedB) > globalParameters["MaxLDS"]: + if (ldsNumBytesAlignedA + ldsNumBytesAlignedB) > depthUConfig.maxLDS: state["LocalReadVectorWidth"] //= 2 else: if state["LocalReadVectorWidth"] == -1: @@ -3488,13 +3490,13 @@ def subCheckLdsBlockSizePerPad(tc, idx): # if User want to control the LDS usage, we may open this para in the future ldsNumBytesReduction = state["LocalSplitU"] * state["MacroTile0"] * state["MacroTile1"] * state["ProblemType"]["ComputeDataType"].numBytes() if state["LocalSplitU"] > 1 else 0 state["LocalSplitUReuseLDS"] = 1 - if ldsNumBytesReduction > globalParameters["MaxLDS"]: - state["LocalSplitUReuseLDS"] = math.ceil(ldsNumBytesReduction / globalParameters["MaxLDS"]) + if ldsNumBytesReduction > depthUConfig.maxLDS: + state["LocalSplitUReuseLDS"] = math.ceil(ldsNumBytesReduction / depthUConfig.maxLDS) # reserve all the LDS to LSU. - ldsNumBytesReduction = globalParameters["MaxLDS"] + ldsNumBytesReduction = depthUConfig.maxLDS # lds max occupancy - ldsSizeOccupancy = globalParameters["DeviceLDS"] // state["MaxOccupancy"] + ldsSizeOccupancy = depthUConfig.deviceLDS // state["MaxOccupancy"] ldsNumBytesOccupancy = ldsSizeOccupancy #print("LdsOffsetB", state["LdsOffsetB"]) @@ -3511,7 +3513,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): if state["1LDSBuffer"] == -1: if ldsNumBytesAB <= max(ldsSizeOccupancy,32768) or \ (state["ProblemType"]["ComputeDataType"].numBytes() * state["MacroTile0"] * state["MacroTile1"] > 32768*4 and \ - not (ldsNumBytesAB > globalParameters["DeviceLDS"])): + not (ldsNumBytesAB > depthUConfig.deviceLDS)): state["1LDSBuffer"] = 0 else: state["1LDSBuffer"] = 1 @@ -3551,7 +3553,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): ldsNumElementsRemapC = max(ldsNumElementsRemapC, ldsNumElementsRemapC * (computeBytes / state["ProblemType"]["DestDataType"].numBytes())) ldsSize = ldsNumElementsRemapC * state["ProblemType"]["DestDataType"].numBytes() if not math.log(state["MacroTile0"],2).is_integer() or \ - ldsSize > globalParameters["MaxLDS"] or \ + ldsSize > depthUConfig.maxLDS or \ state["SourceSwap"] or \ (state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer') or \ state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4 : @@ -3801,8 +3803,8 @@ def calcEpilogueTurns(factorDims: List) -> int: state["LdsNumBytes"] = ldsNumBytes ldsSize = ldsNumBytes - if ldsSize > globalParameters["MaxLDS"]: - reject(state, printRejectionReason, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, globalParameters["MaxLDS"])) + if ldsSize > depthUConfig.maxLDS: + reject(state, printRejectionReason, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, depthUConfig.maxLDS)) state["ValidDepthU"] = False return diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 9c4284608c..511f9e33ea 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -37,8 +37,8 @@ from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ - detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap, isaToGfx, makeDebugConfig -from Tensile.KernelWriter import DebugConfig + detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap, isaToGfx, makeDebugConfig, \ + makeDepthUConfig, DebugConfig, DepthUConfig from Tensile.Toolchain.Assembly import AssemblyToolchain, makeAssemblyToolchain from Tensile.Toolchain.Source import SourceToolchain, makeSourceToolchain from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults @@ -65,6 +65,7 @@ def executeStepsInConfig( isaInfoMap: Dict[str, IsaInfo], cCompiler: str, debugConfig: DebugConfig, + depthUConfig: DepthUConfig, deviceId: int ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -99,7 +100,8 @@ def executeStepsInConfig( outputPath, buildTmpPath, config["ShortNames"], - debugConfig, + debugConfig, + depthUConfig, deviceId, isaInfoMap, ) @@ -126,6 +128,7 @@ def executeStepsInConfig( debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, + depthUConfig, isaInfoMap, ) print1("") @@ -457,6 +460,7 @@ def Tensile(userArgs): config["ShortNames"] = args.shortNames debugConfig = makeDebugConfig(config["GlobalParameters"]) + depthUConfig = makeDepthUConfig(config["GlobalParameters"]) for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) @@ -465,7 +469,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler, debugConfig, device_id) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler, debugConfig, depthUConfig, device_id) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 6f109230b1..38d9633722 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -35,6 +35,7 @@ from Tensile.Common import ( CHeader, DebugConfig, + DepthUConfig, gfxToIsa, HR, IsaInfo, @@ -457,6 +458,7 @@ def generateLogicDataAndSolutions(logicFiles, args, assembler: Assembler, isaInf itertools.repeat(splitGSU), itertools.repeat(printSolutionRejectionReason), itertools.repeat(printIndexAssignmentInfo), + itertools.repeat(DepthUConfig()), itertools.repeat(archs), itertools.repeat(isaInfoMap), itertools.repeat(args["LazyLibraryLoading"]), From 10605c6fe164ded9ae893fd5709ba63c400d6de5 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 00:13:14 -0600 Subject: [PATCH 061/105] 942 tests --- tensilelite/Tensile/ClientWriter.py | 15 ++++++------ tensilelite/Tensile/LibraryIO.py | 23 ++++++++----------- tensilelite/Tensile/Tensile.py | 6 ++--- .../Tensile/TensileCreateLibrary/Run.py | 1 - 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 05e971287c..302f66af5c 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -36,11 +36,12 @@ from . import ClientExecutable from . import LibraryIO from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, IsaInfo, \ - LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR, detectGlobalCurrentISA + LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR, detectGlobalCurrentISA, DepthUConfig from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex from .Contractions import ProblemType as ContractionsProblemType +from Tensile.Toolchain.Component import Assembler class DataInitName(Enum): Zero = 0 @@ -80,7 +81,7 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str, targetGfx: List[str], outputPath: Path, deviceId: int, useShortNames: bool=False): +def main(config, assembler: Assembler, cCompiler: str, isaInfoMap, outputPath: Path, deviceId: int, useShortNames: bool=False): libraryLogicPath = ensurePath(outputPath / LIBRARY_LOGIC_DIR) clientLibraryPath = ensurePath(outputPath / LIBRARY_CLIENT_DIR) @@ -98,7 +99,7 @@ def main(config, cxxCompiler: str, cCompiler: str, targetGfx: List[str], outputP functions = [] functionNames = [] - createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler, targetGfx, useShortNames) + createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, str(assembler.path), isaToGfx(list(isaInfoMap.keys())[0]), useShortNames) subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) coList = glob(os.path.join(clientLibraryPath, "library/*.co")) yamlList = glob(os.path.join(clientLibraryPath, "library/*.yaml")) @@ -106,7 +107,7 @@ def main(config, cxxCompiler: str, cCompiler: str, targetGfx: List[str], outputP clientParametersPaths = [] for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary) \ - = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler, False, False, [targetGfx]) + = LibraryIO.parseLibraryLogicFile(logicFileName, assembler, False, False, False, DepthUConfig(), isaInfoMap, globalParameters["LazyLibraryLoading"]) functions.append((scheduleName, problemType)) functionNames.append("tensile_%s" % (problemType)) problemSizes = ProblemSizesMock(exactLogic) if exactLogic else ProblemSizesMockDummy() @@ -146,10 +147,10 @@ def main(config, cxxCompiler: str, cCompiler: str, targetGfx: List[str], outputP factorDimArgs=factorDimArgs, activationArgs=activationArgs, icacheFlushArgs=icacheFlushArgs, - stepName=str(ProblemType(problemType)), + stepName=str(ProblemType(problemType, False)), stepBaseDir=str(clientLibraryPath), newLibrary=newLibrary, - configBase="ClientParameters_%s"%str(ProblemType(problemType)), + configBase="ClientParameters_%s"%str(ProblemType(problemType, False)), codeObjectFiles=coList, deviceId=deviceId, tileAwareSelection=False, @@ -169,7 +170,7 @@ def main(config, cxxCompiler: str, cCompiler: str, targetGfx: List[str], outputP forBenchmark = False enableTileSelection = False - returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, clientLibraryPath, clientParametersPaths) + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, str(assembler.path), cCompiler, clientLibraryPath, clientParametersPaths) return returncode diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index a87a188568..959502b9e0 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -289,7 +289,6 @@ def parseLibraryLogicFile( printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, depthUConfig: DepthUConfig, - archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool ): @@ -302,7 +301,6 @@ def parseLibraryLogicFile( printSolutionRejectionReason, printIndexAssignmentInfo, depthUConfig, - archs, isaInfoMap, lazyLibraryLoading ) @@ -316,7 +314,6 @@ def parseLibraryLogicData( printSolutionRejectionReason: bool, printIndexAssignmentInfo: bool, depthUConfig: DepthUConfig, - archs, isaInfoMap: Dict[str, IsaInfo], lazyLibraryLoading: bool ): @@ -324,17 +321,15 @@ def parseLibraryLogicData( if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) - supportedISA = [gfxToIsa(a) for a in archs] - - is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") - if not (archs is None) and "ArchitectureName" in data: - if isinstance(archs, List): - if len(archs) > 0 and not archs[0] == "all": - if not (any(is_arch_valid(arch.split(":")[0], data["ArchitectureName"]) for arch in archs)): - return LibraryLogic("", "", None, [], [], None, srcFile) - elif isinstance(archs, str): - if not is_arch_valid(archs.split(":")[0], data["ArchitectureName"]): - return LibraryLogic("", "", None, [], [], None, srcFile) + #is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") + #if not (archs is None) and "ArchitectureName" in data: + # if isinstance(archs, List): + # if len(archs) > 0 and not archs[0] == "all": + # if not (any(is_arch_valid(arch.split(":")[0], data["ArchitectureName"]) for arch in archs)): + # return LibraryLogic("", "", None, [], [], None, srcFile) + # elif isinstance(archs, str): + # if not is_arch_valid(archs.split(":")[0], data["ArchitectureName"]): + # return LibraryLogic("", "", None, [], [], None, srcFile) if "CUCount" not in data: data["CUCount"] = None diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 511f9e33ea..48cbd509a6 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -145,10 +145,10 @@ def executeStepsInConfig( else: libraryClientConfig = {} ClientWriter.main( - libraryClientConfig, - srcToolchain.compiler, + libraryClientConfig, + asmToolchain.assembler, cCompiler, - [isaToGfx(isa) for isa in isaInfoMap.keys()][0], + isaInfoMap, outputPath, deviceId, config["ShortNames"] diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 38d9633722..e1f2771678 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -459,7 +459,6 @@ def generateLogicDataAndSolutions(logicFiles, args, assembler: Assembler, isaInf itertools.repeat(printSolutionRejectionReason), itertools.repeat(printIndexAssignmentInfo), itertools.repeat(DepthUConfig()), - itertools.repeat(archs), itertools.repeat(isaInfoMap), itertools.repeat(args["LazyLibraryLoading"]), ) From 98467464c4481a4204f8a4abf90848af53a622c8 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 00:39:24 -0600 Subject: [PATCH 062/105] Missing arg --- tensilelite/Tensile/BenchmarkProblems.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 2118f56bbb..6cb69a9be3 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -111,13 +111,14 @@ def _generateCustomKernelSolutions( failOnMismatch, assembler: Assembler, debugConfig: DebugConfig, + depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo] ): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, assembler, debugConfig, isaInfoMap) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, assembler, debugConfig, depthUConfig, isaInfoMap) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -364,7 +365,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ not benchmarkStep.customKernelWildcard, asmToolchain.assembler, debugConfig, \ - isaInfoMap) + depthUConfig, isaInfoMap) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions From ef0991fa7e242218dcac7478242c36a73493e5da Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 07:45:03 -0600 Subject: [PATCH 063/105] Separate validParameters --- .../Tensile/Common/GlobalParameters.py | 906 +----------------- tensilelite/Tensile/SolutionStructs.py | 128 ++- 2 files changed, 128 insertions(+), 906 deletions(-) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 89a4b35189..6cc3e33a5d 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -38,6 +38,7 @@ from .Types import IsaVersion, IsaInfo from .Utilities import locateExe, versionIsCompatible, print1, print2, printExit, printWarning, \ verbosity +from .ValidParameters import validParameters startTime = time.time() @@ -332,887 +333,8 @@ } -################################################################################ -# Enumerate Valid Solution Parameters -################################################################################ -validWorkGroups = [] -for numThreads in range(32, 1025, 32): - for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: - for sg0 in range(1, numThreads // nsg + 1): - sg1 = numThreads // nsg // sg0 - if sg0 * sg1 * nsg == numThreads: - workGroup = [sg0, sg1, nsg] - validWorkGroups.append(workGroup) - -validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list( - range(20, 256, 4) -) -validThreadTiles = [] -for i in validThreadTileSides: - for j in validThreadTileSides: - validThreadTiles.append([i, j]) - -validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW") -validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK") -validMacroTileSides = [ - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1024, - 6, - 12, - 24, - 48, - 96, - 192, - 384, - 768, -] -validMacroTiles = [] -validISA = [(0, 0, 0)] -validISA.extend(SUPPORTED_ISA) -depthUs = list(range(2, 1024 + 1, 1)) -for i in validMacroTileSides: - for j in validMacroTileSides: - validMacroTiles.append([i, j]) - -validMFMA = {} -validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] -validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] -validMFMA["4xi8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], - [32, 32, 16, 1], - [16, 16, 32, 1], -] -validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] -validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["C"] = validMFMA["S"] -validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], -] + [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] -validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8"] = validMFMA["F8"] -validMFMA["F8B8"] = validMFMA["F8"] -validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8N"] = validMFMA["F8N"] -validMFMA["F8B8N"] = validMFMA["F8N"] -validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [ - [16, 16, 16, 1], -] -validTT = 32 -validMFMA["_format9"] = [] - -for MFMA in [ - validMFMA["H"], - validMFMA["S"], - validMFMA["B"], - validMFMA["D"], - validMFMA["X"], - validMFMA["F8N"], - validWMMA, -]: - for MI in MFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validMatrixInstructions = ( - [[], [-1]] - + validMFMA["H"] - + validMFMA["S"] - + validMFMA["B"] - + validMFMA["D"] - + validMFMA["B1k"] - + validMFMA["X"] -) -validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] - -validSMFMA = {} -validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8"] = validSMFMA["F8"] -validSMFMA["F8B8"] = validSMFMA["F8"] -validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8N"] = validSMFMA["F8N"] -validSMFMA["F8B8N"] = validSMFMA["F8N"] -validSMFMA["B8F8N"] = validSMFMA["F8N"] -validSMFMA["_format9"] = [] -for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validSMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = ( - validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] -) -# The supported typed GEMM, each entry is (Ti, To, Tc). -# DataType (Ti) = The data-type of the input matrices: A/B -# DestDataType (To) = The data-type of the output matrices: C/D -# ComputeDataType (Tc) = The data-type of computation: alpha/beta: -# Cinternal: basically should == ComputeDataType - -# This is used in SolutionStruct.py::checkIfSupportedGEMMType() -validGEMMTypes = [ - ("H", "H", "H"), - ("S", "S", "S"), - ("D", "D", "D"), - ("C", "C", "C"), - ("Z", "Z", "Z"), - ("H", "H", "S"), - ("H", "S", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I8", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("B8", "B", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), - ("B8F8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), # F8 NANOO - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("B8N", "B", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), - ("B8F8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - -# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: -# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. -# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. -HPATypes = [ - ("H", "S", "S"), - ("H", "H", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - -validParameters = { - # 0: Global read is along parallel direction in thread level, - # each load instruction stride whole threads. - # ----> perp - # | [w0, w0, w1,w1,w2,w2,w3,w3, w0, w0, w1,w1,w2,w2,w3,w3] - # | [ t0,t32] [ ] [ t0,t32] [ ] - # para | [ t1,t33] [ wave 1,2,3 ] [ t1,t33] [ wave 1,2,3 ] - # | [ .., ..] [ ] [ .., ..] [ ] - # | [t31,t63] [ ] [t31,t63] [ ] - # V [-load_1] [-load_2] - # - # 1: Each wave load a block of memory, - # each load instruction stride 64 threads. - # ----> perp - # [ w0, w0, w0, w0, w1,w1,w1,w1, w2,w2,w2,w2, w3,w3,w3,w3] - # | [ t0,t32][ t0,t32] - # para | [ t1,t33][ t1,t33] - # | [ .., ..][ .., ..] - # | [t31,t63][t31,t63] - # V [-load_1][-load_2] - # - # - # 2: Each load instruction spread threads evenly in the perp direction - # ----> perp - # | [w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3] - # | [t0 ] [t0 ] [t32] [t32] - # para | [t1 ] [t1 ] [t33] [t33] - # | [.. ] [.. ] [.. ] [.. ] - # | [t31] [t31] [t63] [t63] - # V [load_1] [load_2] [load_1] [load_2] - # - "WaveSeparateGlobalReadA": [0, 1, 2], - "WaveSeparateGlobalReadB": [0, 1, 2], - # Add an unrolled loop and NGLL loop with swapped GRA and GRB order. - # which may change the tlb thrashing behavior. - "UnrollLoopSwapGlobalReadOrder": [0, 1], - # PrefetchGlobalRead = 1: - # Requires 2X LDS space, and VGPRs for buffering data on way into LDS - # prefetch / double-buffer reads from global memory -> vgprs -> lds. - # - # PrefetchGlobalRead = 2: - # Do another prefetch while writing data from vgpr to lds. - # prefetch / double-buffer reads from global memory -> vgprs --> lds. - # |-> prefetch reads - "PrefetchGlobalRead": [0, 1, 2], - # number of iteration prefetch local reads from lds to VGPRs buffer = PLR - "PrefetchLocalRead": list(range(128 + 1)), - # MatrixInstruction Only - # If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead - # So we can schedule these localReads to the front of the loop - "ClusterLocalRead": [0, 1], - # We use double LDS buffer when PrefetchGlobalRead. - # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] - # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. - # this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer, - # or help to increase Occupancy. - # 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead - # -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS - # Use case: - # SIA2: 1LDSBuffer is set to 1 natively - # SIA3: 1LDSBuffer works only when PGR=True - # TODO: optimize scheduling to support more cases. - "1LDSBuffer": [-1, 0, 1], - # Split the unroll summation into multiple sections and combine the sections - # GSU applies only to the unroll summation dimension - # Set to 0 to disable GSU, kernel code will be generated without GSU support - "GlobalSplitU": list(range(0, 1024 + 1)), - # choose how to do GlobalSplitU - # 1: use atomic operation to accumulate on one buffer - # 2: each GSU group write to each own buffer and accumulate by another kernel - # 3: each GSU group write to each own buffer and accumulate by same kernel - "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], - # don't create a whole copy of the Unroll loop with loads removed - instead - # use buffer limits to suppress global loads and ignore unnecessary ds_reads - "SuppressNoLoadLoop": [False, True], - # For PrefetchGlobalRead=1, create a second copy of the unroll loop with - # the LDS pointer swaps expanded into inline constants for LDS read and write instructions - # This eliminates 4 vector XOR instructions used for pointer swap - "ExpandPointerSwap": [False, True], - # Schedule global reads and global read increments into LocalRead iterations - # Can reduce pressure on local read instruction dispatch queue - # 0=perform global reads at start of instruction loop - # 1=schedule into the local read instruction iterations - "ScheduleGlobalRead": [0, 1], - # Schedule local writes into LocalRead iterations. - # Can reduce pressure on local read instruction dispatch queue - "ScheduleLocalWrite": [0, 1], - # Scheduling algorithm to use for each iteration: - # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, - # followed by local writes, followed by MACs - "ScheduleIterAlg": [0, 1, 2, 3], - # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma - # the purpose of this parameter is to control density of global read instruction scheduling - # Scheduling global read back to back can have better memory efficiency - # However, when full of vmem FIFO, it will block other instruction to be issued - # Range from 0.01 to 32 - # 0.1 means 1 GR per 10 mfma - # 5 means 5 GR per 1 mfma - "GlobalReadPerMfma": [i / 100 for i in range(1, 3200)], - # - # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma - # the purpose of this parameter is to control density of local write instruction scheduling - # In PGR1, we want to schedule local write more denser, so we can have more - # latency to hide global read - # In PGR2, since LW is followed by GR, every LW has same whole loop latency - # to hide global read. We want to schedule LW less denser, can - # avoid full of vmem FIFO. - # Range from 0.01 to 32 - # 0.1 means 1 LW per 10 mfma - # 5 means 5 LW per 1 mfma - # -1 will derived an optimized value internally - # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) - "LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1], - # Interleave alpha scale calculation with beta loads and address calcs - rather - # than as a separate block of instructions - "InterleaveAlpha": [0, 1], - # Create a copy of NoLoadLoop which interleaves the stores with the final mac - # calculation and may perform other optimizations - # 0 = no interleave - # 1 = interleave one stores after required macs have completed execution - # 2 = interleave two stores after required macs have completed execution - "OptNoLoadLoop": [0, 1, 2], - "BufferLoad": [False, True], - "BufferStore": [False, True], - # Attempt to load directly from global memory into Vgpr. - # Assembly only - "DirectToVgprA": [False, True], - "DirectToVgprB": [False, True], - "DirectToVgprSparseMetadata": [False, True], - # Attempt to load directly from global memory into LDS. - # Assembly only - # Requires BufferLoad, assembler support for lds modifier on buffer - # loads (checked automatically), GlobalVectorWidth=1 (this is hw - # requirement) and A/B must not require any transpose. - # DirectToLds reduces load latency and eliminates the - # G2L registers used to stage data. Also replaces the - # local write offset with an SGPR. - # For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs. - # - Requirements for DirectToLds=1: - # GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now) - # TransposeLDS = 1 for TLU=0 case - # DirectToLds support for x1 only for now - "DirectToLds": [False, True], - # Load options: - # (GRO = Global Read Offset) - # BufferLoad=0: - # = Use flat instructions with 64 bit GRO for each load - # + supports sizes up to 2^64 - # - uses many VGPR for addressing - # - uses execmask+compares for edge detection - # - generates extra LDS traffic (could convert flat->global load) - # BufferLoad=1: - # = Use buffer load instructions with 32-bit offset - # + Less VGPRS (32b offset vs 64-bit) needed for addressing - # + Uses hardware buffer limit for edge detection - # - Limited range - the bot-right corner of macro-tile (plus padding=GRVW - # for shift-pointer, if ShiftPtr is required) must be within 2^32. - # ShiftPtrPad = MayShift ? GRWV*BPE : 0 - # For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32 - # For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32 - # These conditions should be checked using Assert - TODO - # = UseSgprForGRO=1: - # + Attempt to use SGPR for Global Read Offsets. - # + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO. - # + Each SGPR stores an offset from base GlobalReadOffset+0. - # - Requirements for UseSgprForGRO=1: - # - BufferLoad=1 - # - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr - # (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO) - # = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit) - # - Requires 4 instructions to move scalar limit and a couple SGPR - # - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize - # = UseInstOffsetForGRO=1: - # + Attempt to use Instruction offset for Global Read Offsets. - # + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature - # - Requirements for UseInstOffsetForGRO=1: - # - BufferLoad=1 - # - DirectToLds=1 - # converting m0 update from LocalWriteAddrSGpr using is usually win - # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseInstOffsetForGRO": [-1, 0, 1], - # Converting VGPR GRO into SGPR GRO is usually a win - # However, the mode may exhaust all available SGPR, in particular for large unroll - # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseSgprForGRO": [-1, 0, 1], - # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes - "Use64bShadowLimit": [True, False], - # Assertion properties - # These provide information or assertions that the problem size meets certain requirements - # for sizes or alignments. The kernel generator can use this information to produce - # a kernel which uses those assertions to produce a faster kernel. - # - # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h - # Kernel generator will assume that the summation size is some multiple of the element size - # and uses this to optimize the kernel. - # This can result in more efficient kernels, but requires runtime checking to ensure the specified - # summation value meets the requirements. - # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) - # - # Optimizations enabled by AssertSummationElementMultiple>1: - # - If >=2 for half: - # - Tail loop loads can be vectorized 2X to use dword - # - Enables asm kernels on V20 - # - Can use DirectToLds for both unroll and tail loops - # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 - # - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128], - # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size - # and uses this to optimize the kernel. - # FreeIndex[0] is usually letter "I" - # (Recommended AF0EM value is 8 for half, 4 for single, 2 for double) - # - # Optimizations enabled by AssertFree0ElementMultiple>1: - # Load optimizations: - # - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO - # - Reduces registers used for address calculations - # - Removes address shift/unshift code - # - UseSgprForGRO will only be enabled if all matrices meet assertion requirements. - # - # Store Optimizations: - # - Can vectorize stores in edge tiles. Vector width can be up to AF0EM. - # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) - # - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree0ElementMultiple": [1, 2, 4, 8, 16], - # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size - # and uses this to optimize the kernel. - # FreeIndex[1] is usually letter "J" - # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) - # Optimizations enabled by AssertFree1ElementMultiple>1: - # - See above AssertFree0ElementMultiple "Load optimizations" - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree1ElementMultiple": [1, 2, 4, 8, 16], - # Assertions that require arithmetic intensity to be specified value. - # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. - # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. - "AssertAIGreaterThanEqual": -1, - "AssertAILessThanEqual": -1, - # Stagger the start summation position of the tiles. - # Elements from the summation dimension are loaded at offsets rather than all starting at 0. - # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping - # for how the specific stagger for a given wg is determined. - # - # The tile assignment C are same as with StaggerOffset=0 ; the difference is the - # order that the summation elements are added. - # GRO will wrap back to the row start when the edge is reached. - # - # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. - # In this case the start of each row of the tile is separated by an exact power-of-2 - # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. - # StaggerU adjusts the start position in the summation (aka 'U') dimension - # to avoid these conflicts. Both A and B matrix start at the adjusted position. - # If >0 specifies the offset in multiples of the macro-tile "unroll" dim - # - Higher values will spread traffic to more channels but provide less L2 re-use. - # - StaggerU and WorkGroupMapping interact and should be tuned together - - # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those - # tiles start reading their summation dim parms. - # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 - "StaggerU": [0, 2, 4, 8, 16, 32, 64], - # Stride in bytes for each staggeru 'click'. - # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so - # each click will start in a new memory channel and spread traffic among the 16 available channels. - # For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points - # in summation dimension, each offset by 256-bytes - provided the tensor dims are large - # enough to support this. - # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. - # (the implementation requires this - the unroll iteration accesses data in steps of - # DepthU*BPE - "StaggerUStride": [-1, 16, 32, 64, 128, 256, 512, 1024, 2048], - # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: - # 0: Use wg0 - # 1: Use wg1 - # 2: Use wg2 - # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) - # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot - # to a different bank since all workgroups still start at same point. - "StaggerUMapping": [0, 1, 2, 3, 4], - # GSU Workgroup Coalesced Ordering - # False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} - # True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} - "GlobalSplitUCoalesced": [False, True], - # GSU Workgroup Mapping - # False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} - # -> workgroups do the summation by tile -> slower GR but faster GW - # True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} - # -> workgroups split up the summation -> faster GR but slower GW - "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], - # 0=don't use magic div (source only) - # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) - # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 - "MagicDivAlg": [0, 1, 2], - # For Block Mapping type: - # 0 : Use hardware-assigned wg number with no remapping. - # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. - # Tensor C always mapped with first free coord as fastest moving - # (Elements in this dimension are sequential in memory. - # - # For 2D nonbatched Matrix this means index order is I, then J - # For 2D batched Matrix this means index order is I, then J, then K. - # - # Then for 2D case: - # - If drawn in row-major format, I is the width and J is the height. - # - WGM determines dimensions of the box used to assign tiles from C - # - WGM is the height of the box (in the J dimension) - # - Given WGM, the box width (in I dim) is determined by number of CUs - # - The box always moves across matrixC in the fastest-moving "I" dim, then - # wraps to next J. TODO - might be useful to change this? - # - # Examples for 2D matrix: - # WGM=8: on CU64 machine this is a square box - # WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping. - # WGM=64: Tall/Skinny - this will cover maximum width in J dimension of C. - # - # Formula for wgSerial: - # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 - "WorkGroupMapping": list( - range(-1024, 1024 + 1) - ), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best - "WorkGroupMappingXCC": [ - 1, - 2, - 4, - 8, - 16, - 32, - ], # change a workgroup's id so that contiguous workgroup can map on same XCC - # -1 : WorkGroupMappingXCCGroup will be set to CU_count at runtime. Please ensure that (CU_count % WGMXCC == 0). - "WorkGroupMappingXCCGroup": list( - range(-1, 1024) - ), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. - "MaxOccupancy": list( - range(1, 40 + 1) - ), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy - "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds - # ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, - # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values - # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs - "ThreadTile": validThreadTiles, - "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 - "WavefrontSize": [32, 64], - # MatrixInstruction: (M x N x K x B) - # XDLOPS tile definition, only valid for gfx908, gfx90a - # MxNxKxB specifies matrix instruction variants - # MxNxB determines the shape of the C tile each instruction worked on - # K determines the unroll depth - # If empty, do not use these instructions - # - # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) - # (Note: MxN means M-by-N in the following comments) - # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: - # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed - # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed - # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels - # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) - # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads - # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 - # Putting it all together: - # - [32, 32, 1, 2, 1, 4, 1, 2, 2] - # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ - # MatrixInst BlkM WT Wave - # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile - # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used - "MatrixInstruction": validMatrixInstructions, - # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. - # MI output data between each threads are along N dims. - # But global memory is along M dim continuous. - # That mean global write between each threads are not continuous. - # Therefore, store performance for MI instruction is poor. - # How StoreRemap works in final store stage: - # 1. Put all thread output data into LDS. - # 2. All thread read data from LDS along M dims. - # (match global Memory continuous direction) - # 3. All thread write out data into global memory. - # 0: Disable StoreRemap (default) - # 1~8: Enable StoreRemap and set the global write vector width - # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) - # -1: Use dwordx2 if support SRVW, or set SRVW to 0 - "StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8], - # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. - "SourceSwap": [False, True], - # Following parameters are designed for store scheduling. - # (store stands for load from C (with beta) and store to C/D) - # - # we want to hide store behind unroll loop - # 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N) - # 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel) - # - # we can hide store behind the other WG's loop by lowering priority of store - # priority of loop is the same as priority of store - # WG0: ???????????????\__ - # |<-- loop --->|<-- store -->|end - # - # WG1: ___________________________/????????????\__ - # |<--------- loop ------------------->|<-- store -->|end - # - # priority of loop is higher than priority of store - # WG0: ???????\____________________ - # |<-- loop --->|<------ store ----->|end - # - # WG1: _____________/?????\__________________ - # |<------- loop -------->|<----- store ---->|end - "StorePriorityOpt": [False, True], - # - # If we issue store in short period of time, kernel will become from compute bound to memory bound - # 0 means issue instructions as many as possible if VGPR available - "NumElementsPerBatchStore": list(range(-1, 256)), - # - # add sync after per batch store in order to store contiguous elements - # add sleep after per batch store in order to distribute store over whole loops - # NOTE: this parameter is highly depends on size_k - # 0 means no sync and sleep - "StoreSyncOpt": list(range(0, 256)), - # - # There are index or address calculation between global instructions. - # issue global instruction b2b has better performance - "GroupLoadStore": [False, True], - # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. - # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. - "MIArchVgpr": [False, True], - # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. - # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. - # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel - # 0 : Standard data-parallel kernel - # 1 : Basic StreamK - # 2 : Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) - # 3 : Two-Tile StreamK with DP before SK tiles - # StreamK kernels can adjust the number of CUs being used. - # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. - # StreamK grid is controlled by setting these enviornment variables: - # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number - # 0 = override disabled (default) - # TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work - # -1 = use prediction model for best performance (not yet implemented) - # 0 = only remainder tiles run in stream-k - # 1+ = remainder + 1 (or more) full grids of tiles run in stream-k (default=1) - # TENSILE_STREAMK_DYNAMIC_GRID selects dynamic grid mode, which automatically limits the number of CUs used: - # 0 = Off, always use all CUs. - # 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count. - # 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power. - # 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs (default). - # Note: dynamic grid coefficients currently apply to gfx942 variants - # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for - # other operations to run in parallel with gemm. - # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. - # 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count. - # The priority of these environment variables is defined as follows: - # TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER - "StreamK": [0, 1, 2, 3], - # Determines if StreamK kernel uses atomics - # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step - # 1: uses atomics to accumulate partial tiles - "StreamKAtomic": [0, 1], - # Enables XCC-based remapping of workgroups, set the value to the number of XCCs - # for the device/configuration being used - # 0: uses default workgroup assignment - # 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs - "StreamKXCCMapping": [0] + list(range(2, 9)), - # Debug settings for stream-k kernels to disable parts of the kernel - # Bit 0: Don't generate fixup code - # Bit 1: Don't generate write to partials code - # Both parts can be disabled together - # 0 = Debug mode off, generate full kernel - # 1 = No fixup - # 2 = No partials - # 3 = Nofixup and no partials - "DebugStreamK": [0, 1, 2, 3], - # Controls desired width (#elements) for loads from global memory -> LDS. - # and eliminates the pointer unshift logic - # -1 : Set GlobalReadVectorWidth = VectorWidth - # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) - # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) - # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) - "GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16], - "GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16], - # Controls desired width (#elements) for loads from LDS -> VGPR. - # -1 : Set LocalReadVectorWidth = VectorWidth - # 1 cannot be used for half type. - # used in combination with TransposeLDS=True - # in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS - # helps optimizing instruction scheduling between MFMA and nonMFMA instructions - # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) - # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) - # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) - "LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16], - # threads should read/write/operate on this many contiguous elements from the C matrix. - # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. - # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) - # Typically the load vector width and store vector width are directly related to the VW. - # The global load width is closely related to the width of local stores so - # GlobalReadVectorWidth also controls local write width. - # Local read width also matches since VectorWidth consec elements must be read - # Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI. - # Using a VW too large which results in >16bytes/thread isn't supported - # For MFMA non SourceSwap: this parameter didn't take effect - # -1 means set vw to largest localReadWidth according to MIWaveTile - "VectorWidthA": [-1, 1, 2, 3, 4, 6, 8], - "VectorWidthB": [-1, 1, 2, 3, 4, 6, 8], - # If 0, store 1 element per instruction. - # If 1, store vector-width elements per instruction. - # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel - "VectorStore": [-1, 0, 1], - # Controls desired width (#elements) for stores from reg to global memory. - # When MatrixInstruciton == None, derived parameter gwvw takes precedence. - # -1 : Set StoreVectorWidth = VectorWidth - "StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8], - # when loading all the data from global into lds requires multiple load instructions, these parameters govern which - # loads will pull which rectangle of data from global into lds - # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible - # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; - # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers - # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension - "NumLoadsCoalescedA": list(range(-1, 64 + 1)), - "NumLoadsCoalescedB": list(range(-1, 64 + 1)), - # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related - # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. - # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. - # subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15... - # So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4; - # and when data is read from global memory the threads read 8 elements along the summation dimension. - # DepthU = LoopUnroll * LocalSplitU = 4*2 in this case - # it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU - # -1 : Only allow GLVW=1 - # -2 : Only allow max(GLVWA,GLVWB) < VW ? - # -3 : Only allow min(GLVWA,GLVWB) < VW ? - "DepthU": depthUs, - # integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting - # performance so this has been deprecated and probably doesn't work - # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) - # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 - "LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8], - # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) - # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). - # Only support LdsBlockSizePerPad >= unrollDepth * BPE - # 0 means disable LdsBlockSizePerPad - "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], - # Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) - # -1 : keep LDS layout same as global fetch dimension for both A and B - # set TLDS = 1 for NN,TN,TT - # set TLDS = 0 for NT - # 0 : coalesced dimension of lds is tile dimension - # 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected - # 2 : coalesced dimension of lds is unroll dimension for both A and B - "TransposeLDS": [-1, 1, 0, 2], - # add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit - # 0: none, 1: glc, 2: slc, 3: glc slc - # For gfx942, sets sc0/sc1/nt bits - # 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1 - "NonTemporalE": list(range(0, 8)), - "NonTemporalD": list(range(0, 8)), - "NonTemporalC": list(range(0, 8)), - "NonTemporalA": list(range(0, 8)), - "NonTemporalB": list(range(0, 8)), - "NonTemporalWS": list(range(0, 8)), - "NonTemporalMetadata": list(range(0, 8)), - "NonTemporal": list(range(-1, 8)), - # Group together unroll iterations inside the unroll loop. - # For example, InnerUnroll=2 will fetch LDS for two unroll iterations - "InnerUnroll": [1, 2, 4, 8, 16, 32, 64], - # Enable CP preload kernel arguments feature - # It can reduce time of loading kernel arguments by s_load. - # It needs new complier and vbios to support this feature. - "PreloadKernArgs": [False, True], - # Kernels should be written in assembly or source - # if assembly, ISA will determine architecture - # if source, Runtime will determine language - # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm - "KernelLanguage": ["Assembly"], - "ISA": validISA, # arch for assembly kernels - # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. - # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section - # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. - # Ex: - # custom.config: - # ProblemType: - # OperationType: GEMM - # etc... - # ThreadTile: [8, 8] - # etc... - # - # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" - # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... - "CustomKernelName": -1, - # Will allow a kernel to be accepted even when checks determine it's not viable. - # Intended for use with custom kernels which have confirmed to be correct - "NoReject": [False, True], - # Debug use only. - "ActivationFused": [False, True], - # True- function call - # False- inline - "ActivationFuncCall": [False, True], - # Alternative implementation for activation function - # Currently only supports GSU == 1 - "ActivationAlt": [False, True], - # Do workgroup reduction. Currently for DBias - "WorkGroupReduction": [False], - # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "Sparse": [0, 1, 2], - # in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds - "ConvertAfterDS": [False, True], - # Force disable shadow init to release more sgpr in preloop - "ForceDisableShadowInit": [False, True], -} - # same parameter for all solution b/c depends only on compiler defaultBenchmarkCommonParameters = [ @@ -1643,29 +765,3 @@ def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, def destinationDictionary[key] = deepcopy(sourceDictionary[key]) else: destinationDictionary[key] = deepcopy(defaultDictionary[key]) - - -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit( - "Invalid parameter name: {}\nValid parameters are {}.".format( - name, sorted(validParameters.keys()) - ) - ) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = ( - " (only first 32 combos printed)\nRefer to Common.py for more info" - if len(validParams[name]) > 32 - else "" - ) - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 3a92c6eec1..a4b4eda68f 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -43,7 +43,7 @@ internalParameters, \ print2, printExit, printWarning, \ validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, \ + roundUp, validWMMA, \ INDEX_CHARS, IsaVersion, SemanticVersion, \ DepthUConfig from Tensile.Toolchain.Component import Assembler @@ -59,6 +59,132 @@ import operator import sys + +# The supported typed GEMM, each entry is (Ti, To, Tc). +# DataType (Ti) = The data-type of the input matrices: A/B +# DestDataType (To) = The data-type of the output matrices: C/D +# ComputeDataType (Tc) = The data-type of computation: alpha/beta: +# Cinternal: basically should == ComputeDataType + +# This is used in SolutionStruct.py::checkIfSupportedGEMMType() +validGEMMTypes = [ + ("H", "H", "H"), + ("S", "S", "S"), + ("D", "D", "D"), + ("C", "C", "C"), + ("Z", "Z", "Z"), + ("H", "H", "S"), + ("H", "S", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I8", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("B8", "B", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), + ("B8F8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), # F8 NANOO + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("B8N", "B", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), + ("B8F8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + +# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: +# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. +# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. +HPATypes = [ + ("H", "S", "S"), + ("H", "H", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + ######################################## # Print a reject message : def reject(state, printSolutionRejectionReason: bool, *args): From 81dfb94d57a2475b457ea6f3bb42626a8d188a94 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 08:00:16 -0600 Subject: [PATCH 064/105] Missing file --- tensilelite/Tensile/Common/ValidParameters.py | 811 ++++++++++++++++++ 1 file changed, 811 insertions(+) create mode 100644 tensilelite/Tensile/Common/ValidParameters.py diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py new file mode 100644 index 0000000000..102ed1e6f0 --- /dev/null +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -0,0 +1,811 @@ +################################################################################ +# +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import math + +from .Architectures import SUPPORTED_ISA +from .Utilities import printExit + +################################################################################ +# Enumerate Valid Solution Parameters +################################################################################ +validWorkGroups = [] +for numThreads in range(32, 1025, 32): + for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: + for sg0 in range(1, numThreads // nsg + 1): + sg1 = numThreads // nsg // sg0 + if sg0 * sg1 * nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + +validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list( + range(20, 256, 4) +) +validThreadTiles = [] +for i in validThreadTileSides: + for j in validThreadTileSides: + validThreadTiles.append([i, j]) + +validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW") +validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK") +validMacroTileSides = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 6, + 12, + 24, + 48, + 96, + 192, + 384, + 768, +] +validMacroTiles = [] +validISA = [(0, 0, 0)] +validISA.extend(SUPPORTED_ISA) +depthUs = list(range(2, 1024 + 1, 1)) +for i in validMacroTileSides: + for j in validMacroTileSides: + validMacroTiles.append([i, j]) + +validMFMA = {} +validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] +validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] +validMFMA["4xi8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], + [32, 32, 16, 1], + [16, 16, 32, 1], +] +validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] +validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["C"] = validMFMA["S"] +validMFMA["Z"] = validMFMA["D"] +validMFMA["I8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], +] + [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] +validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8"] = validMFMA["F8"] +validMFMA["F8B8"] = validMFMA["F8"] +validMFMA["B8F8"] = validMFMA["F8"] +validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8N"] = validMFMA["F8N"] +validMFMA["F8B8N"] = validMFMA["F8N"] +validMFMA["B8F8N"] = validMFMA["F8N"] +validWMMA = [ + [16, 16, 16, 1], +] +validTT = 32 +validMFMA["_format9"] = [] + +for MFMA in [ + validMFMA["H"], + validMFMA["S"], + validMFMA["B"], + validMFMA["D"], + validMFMA["X"], + validMFMA["F8N"], + validWMMA, +]: + for MI in MFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validMatrixInstructions = ( + [[], [-1]] + + validMFMA["H"] + + validMFMA["S"] + + validMFMA["B"] + + validMFMA["D"] + + validMFMA["B1k"] + + validMFMA["X"] +) +validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] + +validSMFMA = {} +validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["I8"] = validSMFMA["4xi8"] +validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8"] = validSMFMA["F8"] +validSMFMA["F8B8"] = validSMFMA["F8"] +validSMFMA["B8F8"] = validSMFMA["F8"] +validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8N"] = validSMFMA["F8N"] +validSMFMA["F8B8N"] = validSMFMA["F8N"] +validSMFMA["B8F8N"] = validSMFMA["F8N"] +validSMFMA["_format9"] = [] +for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: + for MI in SMFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validSMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] +validMatrixInstructions = ( + validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] +) + + +validParameters = { + # 0: Global read is along parallel direction in thread level, + # each load instruction stride whole threads. + # ----> perp + # | [w0, w0, w1,w1,w2,w2,w3,w3, w0, w0, w1,w1,w2,w2,w3,w3] + # | [ t0,t32] [ ] [ t0,t32] [ ] + # para | [ t1,t33] [ wave 1,2,3 ] [ t1,t33] [ wave 1,2,3 ] + # | [ .., ..] [ ] [ .., ..] [ ] + # | [t31,t63] [ ] [t31,t63] [ ] + # V [-load_1] [-load_2] + # + # 1: Each wave load a block of memory, + # each load instruction stride 64 threads. + # ----> perp + # [ w0, w0, w0, w0, w1,w1,w1,w1, w2,w2,w2,w2, w3,w3,w3,w3] + # | [ t0,t32][ t0,t32] + # para | [ t1,t33][ t1,t33] + # | [ .., ..][ .., ..] + # | [t31,t63][t31,t63] + # V [-load_1][-load_2] + # + # + # 2: Each load instruction spread threads evenly in the perp direction + # ----> perp + # | [w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3] + # | [t0 ] [t0 ] [t32] [t32] + # para | [t1 ] [t1 ] [t33] [t33] + # | [.. ] [.. ] [.. ] [.. ] + # | [t31] [t31] [t63] [t63] + # V [load_1] [load_2] [load_1] [load_2] + # + "WaveSeparateGlobalReadA": [0, 1, 2], + "WaveSeparateGlobalReadB": [0, 1, 2], + # Add an unrolled loop and NGLL loop with swapped GRA and GRB order. + # which may change the tlb thrashing behavior. + "UnrollLoopSwapGlobalReadOrder": [0, 1], + # PrefetchGlobalRead = 1: + # Requires 2X LDS space, and VGPRs for buffering data on way into LDS + # prefetch / double-buffer reads from global memory -> vgprs -> lds. + # + # PrefetchGlobalRead = 2: + # Do another prefetch while writing data from vgpr to lds. + # prefetch / double-buffer reads from global memory -> vgprs --> lds. + # |-> prefetch reads + "PrefetchGlobalRead": [0, 1, 2], + # number of iteration prefetch local reads from lds to VGPRs buffer = PLR + "PrefetchLocalRead": list(range(128 + 1)), + # MatrixInstruction Only + # If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead + # So we can schedule these localReads to the front of the loop + "ClusterLocalRead": [0, 1], + # We use double LDS buffer when PrefetchGlobalRead. + # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] + # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. + # this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer, + # or help to increase Occupancy. + # 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead + # -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS + # Use case: + # SIA2: 1LDSBuffer is set to 1 natively + # SIA3: 1LDSBuffer works only when PGR=True + # TODO: optimize scheduling to support more cases. + "1LDSBuffer": [-1, 0, 1], + # Split the unroll summation into multiple sections and combine the sections + # GSU applies only to the unroll summation dimension + # Set to 0 to disable GSU, kernel code will be generated without GSU support + "GlobalSplitU": list(range(0, 1024 + 1)), + # choose how to do GlobalSplitU + # 1: use atomic operation to accumulate on one buffer + # 2: each GSU group write to each own buffer and accumulate by another kernel + # 3: each GSU group write to each own buffer and accumulate by same kernel + "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], + # don't create a whole copy of the Unroll loop with loads removed - instead + # use buffer limits to suppress global loads and ignore unnecessary ds_reads + "SuppressNoLoadLoop": [False, True], + # For PrefetchGlobalRead=1, create a second copy of the unroll loop with + # the LDS pointer swaps expanded into inline constants for LDS read and write instructions + # This eliminates 4 vector XOR instructions used for pointer swap + "ExpandPointerSwap": [False, True], + # Schedule global reads and global read increments into LocalRead iterations + # Can reduce pressure on local read instruction dispatch queue + # 0=perform global reads at start of instruction loop + # 1=schedule into the local read instruction iterations + "ScheduleGlobalRead": [0, 1], + # Schedule local writes into LocalRead iterations. + # Can reduce pressure on local read instruction dispatch queue + "ScheduleLocalWrite": [0, 1], + # Scheduling algorithm to use for each iteration: + # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, + # followed by local writes, followed by MACs + "ScheduleIterAlg": [0, 1, 2, 3], + # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma + # the purpose of this parameter is to control density of global read instruction scheduling + # Scheduling global read back to back can have better memory efficiency + # However, when full of vmem FIFO, it will block other instruction to be issued + # Range from 0.01 to 32 + # 0.1 means 1 GR per 10 mfma + # 5 means 5 GR per 1 mfma + "GlobalReadPerMfma": [i / 100 for i in range(1, 3200)], + # + # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma + # the purpose of this parameter is to control density of local write instruction scheduling + # In PGR1, we want to schedule local write more denser, so we can have more + # latency to hide global read + # In PGR2, since LW is followed by GR, every LW has same whole loop latency + # to hide global read. We want to schedule LW less denser, can + # avoid full of vmem FIFO. + # Range from 0.01 to 32 + # 0.1 means 1 LW per 10 mfma + # 5 means 5 LW per 1 mfma + # -1 will derived an optimized value internally + # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) + "LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1], + # Interleave alpha scale calculation with beta loads and address calcs - rather + # than as a separate block of instructions + "InterleaveAlpha": [0, 1], + # Create a copy of NoLoadLoop which interleaves the stores with the final mac + # calculation and may perform other optimizations + # 0 = no interleave + # 1 = interleave one stores after required macs have completed execution + # 2 = interleave two stores after required macs have completed execution + "OptNoLoadLoop": [0, 1, 2], + "BufferLoad": [False, True], + "BufferStore": [False, True], + # Attempt to load directly from global memory into Vgpr. + # Assembly only + "DirectToVgprA": [False, True], + "DirectToVgprB": [False, True], + "DirectToVgprSparseMetadata": [False, True], + # Attempt to load directly from global memory into LDS. + # Assembly only + # Requires BufferLoad, assembler support for lds modifier on buffer + # loads (checked automatically), GlobalVectorWidth=1 (this is hw + # requirement) and A/B must not require any transpose. + # DirectToLds reduces load latency and eliminates the + # G2L registers used to stage data. Also replaces the + # local write offset with an SGPR. + # For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs. + # - Requirements for DirectToLds=1: + # GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now) + # TransposeLDS = 1 for TLU=0 case + # DirectToLds support for x1 only for now + "DirectToLds": [False, True], + # Load options: + # (GRO = Global Read Offset) + # BufferLoad=0: + # = Use flat instructions with 64 bit GRO for each load + # + supports sizes up to 2^64 + # - uses many VGPR for addressing + # - uses execmask+compares for edge detection + # - generates extra LDS traffic (could convert flat->global load) + # BufferLoad=1: + # = Use buffer load instructions with 32-bit offset + # + Less VGPRS (32b offset vs 64-bit) needed for addressing + # + Uses hardware buffer limit for edge detection + # - Limited range - the bot-right corner of macro-tile (plus padding=GRVW + # for shift-pointer, if ShiftPtr is required) must be within 2^32. + # ShiftPtrPad = MayShift ? GRWV*BPE : 0 + # For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32 + # For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32 + # These conditions should be checked using Assert - TODO + # = UseSgprForGRO=1: + # + Attempt to use SGPR for Global Read Offsets. + # + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO. + # + Each SGPR stores an offset from base GlobalReadOffset+0. + # - Requirements for UseSgprForGRO=1: + # - BufferLoad=1 + # - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr + # (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO) + # = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit) + # - Requires 4 instructions to move scalar limit and a couple SGPR + # - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize + # = UseInstOffsetForGRO=1: + # + Attempt to use Instruction offset for Global Read Offsets. + # + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature + # - Requirements for UseInstOffsetForGRO=1: + # - BufferLoad=1 + # - DirectToLds=1 + # converting m0 update from LocalWriteAddrSGpr using is usually win + # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseInstOffsetForGRO": [-1, 0, 1], + # Converting VGPR GRO into SGPR GRO is usually a win + # However, the mode may exhaust all available SGPR, in particular for large unroll + # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseSgprForGRO": [-1, 0, 1], + # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes + "Use64bShadowLimit": [True, False], + # Assertion properties + # These provide information or assertions that the problem size meets certain requirements + # for sizes or alignments. The kernel generator can use this information to produce + # a kernel which uses those assertions to produce a faster kernel. + # + # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h + # Kernel generator will assume that the summation size is some multiple of the element size + # and uses this to optimize the kernel. + # This can result in more efficient kernels, but requires runtime checking to ensure the specified + # summation value meets the requirements. + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertSummationElementMultiple>1: + # - If >=2 for half: + # - Tail loop loads can be vectorized 2X to use dword + # - Enables asm kernels on V20 + # - Can use DirectToLds for both unroll and tail loops + # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128], + # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[0] is usually letter "I" + # (Recommended AF0EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertFree0ElementMultiple>1: + # Load optimizations: + # - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO + # - Reduces registers used for address calculations + # - Removes address shift/unshift code + # - UseSgprForGRO will only be enabled if all matrices meet assertion requirements. + # + # Store Optimizations: + # - Can vectorize stores in edge tiles. Vector width can be up to AF0EM. + # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree0ElementMultiple": [1, 2, 4, 8, 16], + # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[1] is usually letter "J" + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + # Optimizations enabled by AssertFree1ElementMultiple>1: + # - See above AssertFree0ElementMultiple "Load optimizations" + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree1ElementMultiple": [1, 2, 4, 8, 16], + # Assertions that require arithmetic intensity to be specified value. + # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. + # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. + "AssertAIGreaterThanEqual": -1, + "AssertAILessThanEqual": -1, + # Stagger the start summation position of the tiles. + # Elements from the summation dimension are loaded at offsets rather than all starting at 0. + # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping + # for how the specific stagger for a given wg is determined. + # + # The tile assignment C are same as with StaggerOffset=0 ; the difference is the + # order that the summation elements are added. + # GRO will wrap back to the row start when the edge is reached. + # + # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. + # In this case the start of each row of the tile is separated by an exact power-of-2 + # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. + # StaggerU adjusts the start position in the summation (aka 'U') dimension + # to avoid these conflicts. Both A and B matrix start at the adjusted position. + # If >0 specifies the offset in multiples of the macro-tile "unroll" dim + # - Higher values will spread traffic to more channels but provide less L2 re-use. + # - StaggerU and WorkGroupMapping interact and should be tuned together - + # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those + # tiles start reading their summation dim parms. + # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 + "StaggerU": [0, 2, 4, 8, 16, 32, 64], + # Stride in bytes for each staggeru 'click'. + # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so + # each click will start in a new memory channel and spread traffic among the 16 available channels. + # For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points + # in summation dimension, each offset by 256-bytes - provided the tensor dims are large + # enough to support this. + # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. + # (the implementation requires this - the unroll iteration accesses data in steps of + # DepthU*BPE + "StaggerUStride": [-1, 16, 32, 64, 128, 256, 512, 1024, 2048], + # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: + # 0: Use wg0 + # 1: Use wg1 + # 2: Use wg2 + # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) + # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot + # to a different bank since all workgroups still start at same point. + "StaggerUMapping": [0, 1, 2, 3, 4], + # GSU Workgroup Coalesced Ordering + # False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} + # True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} + "GlobalSplitUCoalesced": [False, True], + # GSU Workgroup Mapping + # False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} + # -> workgroups do the summation by tile -> slower GR but faster GW + # True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} + # -> workgroups split up the summation -> faster GR but slower GW + "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], + # 0=don't use magic div (source only) + # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) + # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 + "MagicDivAlg": [0, 1, 2], + # For Block Mapping type: + # 0 : Use hardware-assigned wg number with no remapping. + # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. + # Tensor C always mapped with first free coord as fastest moving + # (Elements in this dimension are sequential in memory. + # + # For 2D nonbatched Matrix this means index order is I, then J + # For 2D batched Matrix this means index order is I, then J, then K. + # + # Then for 2D case: + # - If drawn in row-major format, I is the width and J is the height. + # - WGM determines dimensions of the box used to assign tiles from C + # - WGM is the height of the box (in the J dimension) + # - Given WGM, the box width (in I dim) is determined by number of CUs + # - The box always moves across matrixC in the fastest-moving "I" dim, then + # wraps to next J. TODO - might be useful to change this? + # + # Examples for 2D matrix: + # WGM=8: on CU64 machine this is a square box + # WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping. + # WGM=64: Tall/Skinny - this will cover maximum width in J dimension of C. + # + # Formula for wgSerial: + # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 + "WorkGroupMapping": list( + range(-1024, 1024 + 1) + ), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best + "WorkGroupMappingXCC": [ + 1, + 2, + 4, + 8, + 16, + 32, + ], # change a workgroup's id so that contiguous workgroup can map on same XCC + # -1 : WorkGroupMappingXCCGroup will be set to CU_count at runtime. Please ensure that (CU_count % WGMXCC == 0). + "WorkGroupMappingXCCGroup": list( + range(-1, 1024) + ), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. + "MaxOccupancy": list( + range(1, 40 + 1) + ), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy + "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds + # ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, + # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values + # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs + "ThreadTile": validThreadTiles, + "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 + "WavefrontSize": [32, 64], + # MatrixInstruction: (M x N x K x B) + # XDLOPS tile definition, only valid for gfx908, gfx90a + # MxNxKxB specifies matrix instruction variants + # MxNxB determines the shape of the C tile each instruction worked on + # K determines the unroll depth + # If empty, do not use these instructions + # + # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + # (Note: MxN means M-by-N in the following comments) + # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + # Putting it all together: + # - [32, 32, 1, 2, 1, 4, 1, 2, 2] + # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + # MatrixInst BlkM WT Wave + # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + "MatrixInstruction": validMatrixInstructions, + # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. + # MI output data between each threads are along N dims. + # But global memory is along M dim continuous. + # That mean global write between each threads are not continuous. + # Therefore, store performance for MI instruction is poor. + # How StoreRemap works in final store stage: + # 1. Put all thread output data into LDS. + # 2. All thread read data from LDS along M dims. + # (match global Memory continuous direction) + # 3. All thread write out data into global memory. + # 0: Disable StoreRemap (default) + # 1~8: Enable StoreRemap and set the global write vector width + # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) + # -1: Use dwordx2 if support SRVW, or set SRVW to 0 + "StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8], + # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. + "SourceSwap": [False, True], + # Following parameters are designed for store scheduling. + # (store stands for load from C (with beta) and store to C/D) + # + # we want to hide store behind unroll loop + # 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N) + # 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel) + # + # we can hide store behind the other WG's loop by lowering priority of store + # priority of loop is the same as priority of store + # WG0: ???????????????\__ + # |<-- loop --->|<-- store -->|end + # + # WG1: ___________________________/????????????\__ + # |<--------- loop ------------------->|<-- store -->|end + # + # priority of loop is higher than priority of store + # WG0: ???????\____________________ + # |<-- loop --->|<------ store ----->|end + # + # WG1: _____________/?????\__________________ + # |<------- loop -------->|<----- store ---->|end + "StorePriorityOpt": [False, True], + # + # If we issue store in short period of time, kernel will become from compute bound to memory bound + # 0 means issue instructions as many as possible if VGPR available + "NumElementsPerBatchStore": list(range(-1, 256)), + # + # add sync after per batch store in order to store contiguous elements + # add sleep after per batch store in order to distribute store over whole loops + # NOTE: this parameter is highly depends on size_k + # 0 means no sync and sleep + "StoreSyncOpt": list(range(0, 256)), + # + # There are index or address calculation between global instructions. + # issue global instruction b2b has better performance + "GroupLoadStore": [False, True], + # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. + # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. + "MIArchVgpr": [False, True], + # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. + # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. + # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel + # 0 : Standard data-parallel kernel + # 1 : Basic StreamK + # 2 : Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) + # 3 : Two-Tile StreamK with DP before SK tiles + # StreamK kernels can adjust the number of CUs being used. + # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. + # StreamK grid is controlled by setting these enviornment variables: + # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number + # 0 = override disabled (default) + # TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work + # -1 = use prediction model for best performance (not yet implemented) + # 0 = only remainder tiles run in stream-k + # 1+ = remainder + 1 (or more) full grids of tiles run in stream-k (default=1) + # TENSILE_STREAMK_DYNAMIC_GRID selects dynamic grid mode, which automatically limits the number of CUs used: + # 0 = Off, always use all CUs. + # 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count. + # 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power. + # 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs (default). + # Note: dynamic grid coefficients currently apply to gfx942 variants + # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for + # other operations to run in parallel with gemm. + # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. + # 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count. + # The priority of these environment variables is defined as follows: + # TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER + "StreamK": [0, 1, 2, 3], + # Determines if StreamK kernel uses atomics + # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step + # 1: uses atomics to accumulate partial tiles + "StreamKAtomic": [0, 1], + # Enables XCC-based remapping of workgroups, set the value to the number of XCCs + # for the device/configuration being used + # 0: uses default workgroup assignment + # 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs + "StreamKXCCMapping": [0] + list(range(2, 9)), + # Debug settings for stream-k kernels to disable parts of the kernel + # Bit 0: Don't generate fixup code + # Bit 1: Don't generate write to partials code + # Both parts can be disabled together + # 0 = Debug mode off, generate full kernel + # 1 = No fixup + # 2 = No partials + # 3 = Nofixup and no partials + "DebugStreamK": [0, 1, 2, 3], + # Controls desired width (#elements) for loads from global memory -> LDS. + # and eliminates the pointer unshift logic + # -1 : Set GlobalReadVectorWidth = VectorWidth + # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) + # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) + # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) + "GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16], + "GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16], + # Controls desired width (#elements) for loads from LDS -> VGPR. + # -1 : Set LocalReadVectorWidth = VectorWidth + # 1 cannot be used for half type. + # used in combination with TransposeLDS=True + # in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS + # helps optimizing instruction scheduling between MFMA and nonMFMA instructions + # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) + # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) + # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) + "LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16], + # threads should read/write/operate on this many contiguous elements from the C matrix. + # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. + # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) + # Typically the load vector width and store vector width are directly related to the VW. + # The global load width is closely related to the width of local stores so + # GlobalReadVectorWidth also controls local write width. + # Local read width also matches since VectorWidth consec elements must be read + # Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI. + # Using a VW too large which results in >16bytes/thread isn't supported + # For MFMA non SourceSwap: this parameter didn't take effect + # -1 means set vw to largest localReadWidth according to MIWaveTile + "VectorWidthA": [-1, 1, 2, 3, 4, 6, 8], + "VectorWidthB": [-1, 1, 2, 3, 4, 6, 8], + # If 0, store 1 element per instruction. + # If 1, store vector-width elements per instruction. + # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel + "VectorStore": [-1, 0, 1], + # Controls desired width (#elements) for stores from reg to global memory. + # When MatrixInstruciton == None, derived parameter gwvw takes precedence. + # -1 : Set StoreVectorWidth = VectorWidth + "StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8], + # when loading all the data from global into lds requires multiple load instructions, these parameters govern which + # loads will pull which rectangle of data from global into lds + # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible + # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; + # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers + # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension + "NumLoadsCoalescedA": list(range(-1, 64 + 1)), + "NumLoadsCoalescedB": list(range(-1, 64 + 1)), + # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related + # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. + # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. + # subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15... + # So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4; + # and when data is read from global memory the threads read 8 elements along the summation dimension. + # DepthU = LoopUnroll * LocalSplitU = 4*2 in this case + # it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU + # -1 : Only allow GLVW=1 + # -2 : Only allow max(GLVWA,GLVWB) < VW ? + # -3 : Only allow min(GLVWA,GLVWB) < VW ? + "DepthU": depthUs, + # integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting + # performance so this has been deprecated and probably doesn't work + # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) + # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 + "LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8], + # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) + # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). + # Only support LdsBlockSizePerPad >= unrollDepth * BPE + # 0 means disable LdsBlockSizePerPad + "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], + # Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) + # -1 : keep LDS layout same as global fetch dimension for both A and B + # set TLDS = 1 for NN,TN,TT + # set TLDS = 0 for NT + # 0 : coalesced dimension of lds is tile dimension + # 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected + # 2 : coalesced dimension of lds is unroll dimension for both A and B + "TransposeLDS": [-1, 1, 0, 2], + # add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit + # 0: none, 1: glc, 2: slc, 3: glc slc + # For gfx942, sets sc0/sc1/nt bits + # 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1 + "NonTemporalE": list(range(0, 8)), + "NonTemporalD": list(range(0, 8)), + "NonTemporalC": list(range(0, 8)), + "NonTemporalA": list(range(0, 8)), + "NonTemporalB": list(range(0, 8)), + "NonTemporalWS": list(range(0, 8)), + "NonTemporalMetadata": list(range(0, 8)), + "NonTemporal": list(range(-1, 8)), + # Group together unroll iterations inside the unroll loop. + # For example, InnerUnroll=2 will fetch LDS for two unroll iterations + "InnerUnroll": [1, 2, 4, 8, 16, 32, 64], + # Enable CP preload kernel arguments feature + # It can reduce time of loading kernel arguments by s_load. + # It needs new complier and vbios to support this feature. + "PreloadKernArgs": [False, True], + # Kernels should be written in assembly or source + # if assembly, ISA will determine architecture + # if source, Runtime will determine language + # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm + "KernelLanguage": ["Assembly"], + "ISA": validISA, # arch for assembly kernels + # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. + # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section + # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. + # Ex: + # custom.config: + # ProblemType: + # OperationType: GEMM + # etc... + # ThreadTile: [8, 8] + # etc... + # + # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" + # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... + "CustomKernelName": -1, + # Will allow a kernel to be accepted even when checks determine it's not viable. + # Intended for use with custom kernels which have confirmed to be correct + "NoReject": [False, True], + # Debug use only. + "ActivationFused": [False, True], + # True- function call + # False- inline + "ActivationFuncCall": [False, True], + # Alternative implementation for activation function + # Currently only supports GSU == 1 + "ActivationAlt": [False, True], + # Do workgroup reduction. Currently for DBias + "WorkGroupReduction": [False], + # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "Sparse": [0, 1, 2], + # in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds + "ConvertAfterDS": [False, True], + # Force disable shadow init to release more sgpr in preloop + "ForceDisableShadowInit": [False, True], +} + + +def checkParametersAreValid(param, validParams): + """Ensures paramaters in params exist and have valid values as specified by validParames""" + (name, values) = param + if name == "ProblemSizes": + return + elif name == "InternalSupportParams": + return + + if name not in validParams: + printExit( + "Invalid parameter name: {}\nValid parameters are {}.".format( + name, sorted(validParameters.keys()) + ) + ) + + for value in values: + if validParams[name] != -1 and value not in validParams[name]: + msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." + msgExt = ( + " (only first 32 combos printed)\nRefer to Common.py for more info" + if len(validParams[name]) > 32 + else "" + ) + printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) From 02a0f2f7c0ce4ca330556a4479387b641b0f3325 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 09:32:16 -0600 Subject: [PATCH 065/105] add ValidParameters to common module --- tensilelite/Tensile/Common/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index 2addea2cd1..d62b80c1c4 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -8,3 +8,4 @@ from .Parallel import * from .Types import * from .Utilities import * +from .ValidParameters import * From 90b5e8f911b58669455379de2e60f8e741d625cf Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 16:23:53 +0000 Subject: [PATCH 066/105] Isolated naming functions --- tensilelite/Tensile/BenchmarkProblems.py | 14 +- .../Tensile/Common/GlobalParameters.py | 5 +- tensilelite/Tensile/Common/ValidParameters.py | 7 +- tensilelite/Tensile/Common/__init__.py | 1 + tensilelite/Tensile/KernelWriter.py | 40 +-- tensilelite/Tensile/KernelWriterAssembly.py | 4 +- tensilelite/Tensile/LibraryLogic.py | 28 +- tensilelite/Tensile/SolutionLibrary.py | 8 +- .../Tensile/SolutionSelectionLibrary.py | 6 +- tensilelite/Tensile/SolutionStructs.py | 272 +----------------- .../Tensile/TensileCreateLibrary/Run.py | 74 +++-- tensilelite/Tensile/Toolchain/Assembly.py | 10 +- 12 files changed, 93 insertions(+), 376 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 6cb69a9be3..e04e12695b 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -47,7 +47,7 @@ from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx, \ - DepthUConfig + DepthUConfig, getMinNaming, getNameMin, getSerialNaming, getNameFull def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ @@ -194,8 +194,8 @@ def writeBenchmarkFiles( kernelHelperObjs.append(ko) kernelHelperNames.add(kname) - kernelSerialNaming = Solution.getSerialNaming(kernels) - kernelMinNaming = Solution.getMinNaming(kernels) + kernelSerialNaming = getSerialNaming(kernels) + kernelMinNaming = getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( kernelMinNaming, kernelSerialNaming, @@ -385,7 +385,7 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize printExit(msg) for solution in solutions: - print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution, debugConfig.splitGSU))) + print2("# ({}:{}) {}".format(0, 0, getNameFull(solution, debugConfig.splitGSU))) print2(HR) # write benchmarkFiles @@ -412,12 +412,12 @@ def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSize .format(len(solutions), prevCount )) # add SolutionIndex and SolutionNameMin into benchmark yaml - solutionMinNaming = Solution.getMinNaming(solutions) + solutionMinNaming = getMinNaming(solutions) for i in range(0, len(solutions)): solution = solutions[i] solution["SolutionIndex"] = i - solution["SolutionNameMin"] = Solution.getNameMin(solution, solutionMinNaming, debugConfig.splitGSU) - solution["KernelNameMin"] = Solution.getNameMin(solution, solutionMinNaming, debugConfig.splitGSU, True) + solution["SolutionNameMin"] = getNameMin(solution, solutionMinNaming, debugConfig.splitGSU) + solution["KernelNameMin"] = getNameMin(solution, solutionMinNaming, debugConfig.splitGSU, True) else: solutions = None print1("# Using cached solution data") diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 6cc3e33a5d..090e69dbcd 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -305,6 +305,7 @@ globalParameters["UseEffLike"] = True # Set to False to use winnerGFlops as the performance metric # Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen +# we should do this here... defaultGlobalParameters = deepcopy(globalParameters) @@ -332,10 +333,6 @@ "UseUniversalArgs": True, } - - - - # same parameter for all solution b/c depends only on compiler defaultBenchmarkCommonParameters = [ {"InnerUnroll": [1]}, diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index 102ed1e6f0..571a2c6c40 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ # ################################################################################ -import math +import math from .Architectures import SUPPORTED_ISA from .Utilities import printExit @@ -748,7 +748,8 @@ # if source, Runtime will determine language # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm "KernelLanguage": ["Assembly"], - "ISA": validISA, # arch for assembly kernels + # We set ISA in multiple places + "ISA": validISA, # arch for assembly kernels # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index d62b80c1c4..b0a4e616f0 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -5,6 +5,7 @@ # Dunder variables are not exported via `*` from .GlobalParameters import * from .GlobalParameters import __version__ +from .Naming import * from .Parallel import * from .Types import * from .Utilities import * diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 064385dd65..15c66e4e70 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -32,12 +32,11 @@ from .TensilePass import TensilePass, TensilePassOptions from .Component import Component, LraTileProperties from .Components.Signature import UserArgumentsInfo -from .CustomKernels import isCustomKernelConfig from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ - INDEX_CHARS, MAX_FILENAME_LENGTH, IsaVersion + INDEX_CHARS, IsaVersion, getKernelName from Tensile.Toolchain.Component import Assembler import abc @@ -2923,7 +2922,7 @@ def _initKernel(self, kernel, tensorParametersA, tensorParametersB): self.ti.getArchCaps self.consts = ConstValues() - self.states = StateValues(version=version, kernel=kernel, kernelName=self.getKernelName(kernel)) + self.states = StateValues(version=version, kernel=kernel, kernelName=getKernelName(self.kernelMinNaming, self.debugConfig.splitGSU, kernel)) self.vgprs = StateVgprs() self.sgprs = collections.OrderedDict() self.codes = CodeModules() @@ -4978,23 +4977,6 @@ def _syncThreads(self, kernel, comment="", skipForceWaitcnt0=False): # ############################################################################## - def _shortenFileBase(self, kernel): - base = self.getKernelName(kernel) - if len(base) <= MAX_FILENAME_LENGTH: - return base - - import hashlib - import base64 - - pivot = MAX_FILENAME_LENGTH * 3 // 4 - firstPart = base[:pivot] - secondPart = base[pivot:] - - secondHash = hashlib.sha256(secondPart.encode()).digest() - secondPart = base64.b64encode(secondHash, b'_-').decode() - - return firstPart + secondPart - def _getKernelSource(self, kernel: Solution): """ @@ -5017,22 +4999,6 @@ def _getKernelSource(self, kernel: Solution): return fileString - ############################################################################## - # get kernel name - ############################################################################## - def getKernelFileBase(self, useShortNames: bool, kernel): - if isCustomKernelConfig(kernel): - fileBase = kernel["CustomKernelName"] - elif useShortNames: - fileBase = Solution.getNameSerial(kernel, self.kernelSerialNaming) - else: - fileBase = self._shortenFileBase(kernel) - return fileBase - - def getKernelName(self, kernel): - kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, self.debugConfig.splitGSU, True) - return kernelName - @abc.abstractmethod def getSourceFileString(self, kernel) -> Tuple[int, str]: """ @@ -5048,7 +5014,7 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: pass def getHeaderFileString(self, kernel): - kernelName = self.getKernelName(kernel) + kernelName = getKernelName(self.kernelMinNaming, self.debugConfig.splitGSU, kernel) fileString = "" # CHeader fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 293bebd7b1..07475dcb00 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -49,7 +49,7 @@ from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, \ - SemanticVersion, IsaVersion, IsaInfo + SemanticVersion, IsaVersion, IsaInfo, getKernelFileBase from Tensile.Toolchain.Component import Assembler from math import ceil, log, floor @@ -94,7 +94,7 @@ def __init__( def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): - kernelName = self.getKernelFileBase(useShortNames, kernel) + kernelName = getKernelFileBase(useShortNames, self.kernelSerialNaming, kernel) with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: rocmVersion = self.assembler.rocm_version if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650): diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index dc5bcc1427..27dd5fb3bb 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -26,7 +26,7 @@ from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig, getMinNaming, getNameMin from .SolutionStructs import Solution from . import LibraryIO from . import SolutionSelectionLibrary @@ -71,13 +71,13 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL solutions = problemSizeGroup[4] problemSizesList.append(problemSizes) solutionsList.append(solutions) - solutionMinNaming = Solution.getMinNaming(solutions) + solutionMinNaming = getMinNaming(solutions) print1("# Read: %s" % (solutionsFileName)) print2("# ProblemSizes: %s" % problemSizes) print2("# Solutions:") solutionIdx = 0 for solution in solutions: - print2("# (%u) %s" % (solutionIdx, Solution.getNameMin(solution, \ + print2("# (%u) %s" % (solutionIdx, getNameMin(solution, \ solutionMinNaming, splitGSU))) solutionIdx += 1 print2(HR) @@ -127,9 +127,9 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL for i in range(0, len(logicAnalyzer.solutions)): s = logicAnalyzer.solutions[i] s["SolutionIndex"] = i - s["SolutionNameMin"] = Solution.getNameMin(s, solutionMinNaming, splitGSU) - s["KernelNameMin"] = Solution.getNameMin(s, solutionMinNaming, splitGSU, True) - print1("(%2u) %s : %s" % (i, Solution.getNameMin(s, solutionMinNaming, splitGSU), Solution.getNameFull(s, splitGSU))) + s["SolutionNameMin"] = getNameMin(s, solutionMinNaming, splitGSU) + s["KernelNameMin"] = getNameMin(s, solutionMinNaming, splitGSU, True) + print1("(%2u) %s : %s" % (i, getNameMin(s, solutionMinNaming, splitGSU), Solution.getNameFull(s, splitGSU))) if enableTileSelection: validSelectionSolutions = SolutionSelectionLibrary.analyzeSolutionSelection(problemType, selectionFileNameList, \ @@ -161,8 +161,8 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL (validSolution, validSolutionInfo) = validSelectionSolution selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU, True) + validSolution["SolutionNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) @@ -292,11 +292,11 @@ def __init__(self, problemType, problemSizesList, solutionsList, \ self.solutionGroupMap[solutionGroupIdx][solutionIdx] = sIdx progressBar.increment() self.numSolutions = len(self.solutions) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionNames.append(getNameMin(solution, \ self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) @@ -1119,11 +1119,11 @@ def removeSolution(self, removeSolutionIdx): for i in range(0, oldNumSolutions): if i != removeSolutionIdx: self.solutions.append(oldSolutions[i]) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionNames.append(getNameMin(solution, \ self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) @@ -1170,11 +1170,11 @@ def pruneSolutions(self, keepSolutions): else: removeSolutionIdxList.append(i) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ + self.solutionNames.append(getNameMin(solution, \ self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index c97e02091d..aca9ffb01d 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -29,7 +29,7 @@ from . import Hardware from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state, IsaInfo, gfxToIsa, DepthUConfig +from .Common import state, IsaInfo, gfxToIsa, DepthUConfig, getMinNaming, getNameMin class SingleSolutionLibrary: Tag = "Single" @@ -559,11 +559,11 @@ def state(self): def applyNaming(self, splitGSU: bool, naming=None): if naming is None: kernels = itertools.chain(s.originalSolution.getKernels() for s in self.solutions.values()) - naming = OriginalSolution.getMinNaming(kernels) + naming = getMinNaming(kernels) for s in list(self.solutions.values()): - s.name = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU) - s.kernelName = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU, True) + s.name = getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU) + s.kernelName = getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU, True) def remapSolutionIndicesStartingFrom(self, curIndex): reIndexMap = {} diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py index 6df4ea21f0..d3944d309c 100644 --- a/tensilelite/Tensile/SolutionSelectionLibrary.py +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -22,7 +22,7 @@ # ################################################################################ -from .SolutionStructs import Solution +from Tensile.Common import getNameMin import csv @@ -90,8 +90,8 @@ def updateValidSolutions(validSolutions, analyzerSolutions, solutionMinNaming): selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) splitGSU = False # this is a reminder that we need to add this in to the function signature - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, splitGSU, True) + validSolution["SolutionNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index a4b4eda68f..c7bdb8d378 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -33,8 +33,6 @@ from .KernelWriterReduction import KernelWriterReduction from .Activation import ActivationType - -from .CustomKernels import isCustomKernelConfig from .AsmStoreState import VectorDataTypes from .Common import assignParameterWithDefault, IsaInfo, \ @@ -42,10 +40,10 @@ defaultInternalSupportParams, \ internalParameters, \ print2, printExit, printWarning, \ - validMFMA, validSMFMA, validParameters, \ + validMFMA, validSMFMA, \ roundUp, validWMMA, \ INDEX_CHARS, IsaVersion, SemanticVersion, \ - DepthUConfig + DepthUConfig, getNameFull from Tensile.Toolchain.Component import Assembler from collections import OrderedDict @@ -1252,7 +1250,7 @@ def __init__( assembler.rocm_version, depthUConfig, ) - self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None + self._name = config["CustomKernelName"] if "CustomKernelName" in config and config["CustomKernelName"] else None self.initHelperKernelObjects(targetIsas) @@ -4205,218 +4203,6 @@ def calcEpilogueTurns(factorDims: List) -> int: #print("Force to Disable PreloadKernArgs since this hipcc version doesn't support",) state["PreloadKernArgs"] = 0 - ######################################## - # create a dictionary with booleans on whether to include parameter in name - @staticmethod - def getMinNaming(objs): - nonCKObjs = [obj for obj in objs if not isCustomKernelConfig(obj)] - - # early return - if len(nonCKObjs) == 0: - return {} - - # determine keys - requiredParameters = {} - if isinstance(nonCKObjs[0], Solution): - keys = list(nonCKObjs[0]._state.keys()) - else: - keys = list(nonCKObjs[0].keys()) - # only 1, rather than name being nothing, it'll be everything - if len(nonCKObjs) == 1: - for key in keys: - if key in list(validParameters.keys()): - requiredParameters[key] = False - else: - for key in keys: - required = False - if key in list(validParameters.keys()): - for i in range(1, len(nonCKObjs)): - if nonCKObjs[0][key] != nonCKObjs[i][key]: - required = True - break - if required: - requiredParameters[key] = True - else: - requiredParameters[key] = False - - requiredParameters["GlobalSplitU"] = True - requiredParameters["WorkGroupMapping"] = True - - if "MatrixInstM" in nonCKObjs[0]._state: - # Use MIWaveGroup and MIWaveTile instead of WG and MT - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - - requiredParameters["ProblemType"] = False # always prepended - requiredParameters["MacroTile0"] = False # always prepended - requiredParameters["MacroTile1"] = False # always prepended - requiredParameters["DepthU"] = False # always prepended - requiredParameters["MatrixInstruction"] = False # always prepended - requiredParameters["MatrixInstM"] = False # always prepended - requiredParameters["MatrixInstN"] = False # always prepended - requiredParameters["MatrixInstK"] = False # always prepended - requiredParameters["MatrixInstB"] = False # always prepended - requiredParameters["MatrixInstBM"] = False # always prepended - requiredParameters["MatrixInstBN"] = False # always prepended - requiredParameters["CustomKernelName"] = False # Will not affect naming - - requiredParameters["Kernel"] = True # distinguish kernels from solutions - # for single-source compilation - return requiredParameters - - ######################################## - @ staticmethod - def getKeyNoInternalArgs(state, splitGSU: bool): - state_copy = deepcopy(state) - - state_copy["ProblemType"]["GroupedGemm"] = False - - if splitGSU: - state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] - elif state["GlobalSplitU"] > 0: - state_copy["GlobalSplitU"] = "M" - state_copy["WorkGroupMapping"] = "M" - state_copy["WorkGroupMappingXCC"] = "M" - state_copy["WorkGroupMappingXCCGroup"] = "M" - state_copy["StaggerU"] = "M" - state_copy["StaggerUStride"] = "M" - state_copy["StaggerUMapping"] = "M" - state_copy["GlobalSplitUCoalesced"] = "M" - state_copy["GlobalSplitUWorkGroupMappingRoundRobin"] = "M" - - return state_copy - - @ staticmethod - def getNameFull(state, splitGSU: bool): - requiredParameters = {} - for key in state: - if key in list(validParameters.keys()): - requiredParameters[key] = True - if "MatrixInstM" in state: - # Use MIWaveGroup and MIWaveTile instead of WG and MT - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - return Solution.getNameMin(state, requiredParameters, splitGSU) - - ######################################## - # Get Name Min - @ staticmethod - def getNameMin(state, requiredParameters, splitGSU: bool, ignoreInternalArgs = False): - - if isCustomKernelConfig(state): - return state["CustomKernelName"] - - components = [] - - backup = state["ProblemType"]["GroupedGemm"] - if ignoreInternalArgs: - state["ProblemType"]["GroupedGemm"] = False - - if "ProblemType" in state: - components.append(f'{str(state["ProblemType"])}') - # name += str(state["ProblemType"]) + "_" - - if ignoreInternalArgs: - state["ProblemType"]["GroupedGemm"] = backup - - if "MacroTile0" in state \ - and "MacroTile1" in state \ - and "DepthU" in state: - components.append(f'{Solution.getParameterNameAbbreviation("MacroTile")}{state["MacroTile0"]}x{state["MacroTile1"]}x{state["DepthU"]}') - - if "MatrixInstM" in state: - components.append(f'{Solution.getParameterNameAbbreviation("MatrixInstruction")}{state["MatrixInstM"]}x{state["MatrixInstN"]}x{state["MatrixInstB"]}') - - backup = state["GlobalSplitU"] - - if ignoreInternalArgs: - if splitGSU: - state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] - elif state["GlobalSplitU"] > 0: - requiredParameters["GlobalSplitU"] = False - requiredParameters["WorkGroupMapping"] = False - requiredParameters["WorkGroupMappingXCC"] = False - requiredParameters["WorkGroupMappingXCCGroup"] = False - requiredParameters["StaggerU"] = False - requiredParameters["StaggerUStride"] = False - requiredParameters["StaggerUMapping"] = False - requiredParameters["GlobalSplitUCoalesced"] = False - requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = False - - useWaveTile, useThreadTile = requiredParameters.get("MIWaveTile", False), requiredParameters.get("ThreadTile", False) - - if 'MatrixInstM' in state: - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - else: - requiredParameters["MIWaveTile"] = False - requiredParameters["ThreadTile"] = True - - components.append('SN') - for key in sorted(state.keys()): - if key in requiredParameters and key[0] != '_': - if requiredParameters[key] and key != "CustomKernelName": - components.append(f'{Solution.getParameterNameAbbreviation(key)}{Solution.getParameterValueAbbreviation(key, state[key])}') - - state["GlobalSplitU"] = backup - requiredParameters["GlobalSplitU"] = True - requiredParameters["WorkGroupMapping"] = True - requiredParameters["WorkGroupMappingXCC"] = True - requiredParameters["WorkGroupMappingXCCGroup"] = True - requiredParameters["StaggerU"] = True - requiredParameters["StaggerUStride"] = True - requiredParameters["StaggerUMapping"] = True - requiredParameters["GlobalSplitUCoalesced"] = True - requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = True - requiredParameters["MIWaveTile"] = useWaveTile - requiredParameters["ThreadTile"] = useThreadTile - - return '_'.join(components) - - ######################################## - # create a dictionary of lists of parameter values - @staticmethod - def getSerialNaming(objs): - data = {} - for obj in objs: - for paramName in sorted(obj.keys()): - if paramName in validParameters.keys(): - paramValue = obj[paramName] - if paramName in data: - if paramValue not in data[paramName]: - data[paramName].append(paramValue) - else: - data[paramName] = [ paramValue ] - maxObjs = 1 - for paramName in data: - if not isinstance(data[paramName][0], dict): - data[paramName] = sorted(data[paramName]) - maxObjs *= len(data[paramName]) - numDigits = len(str(maxObjs)) - return [ data, numDigits ] - - ######################################## - # Get Name Serial - @ staticmethod - def getNameSerial(state, serialNaming): - data = serialNaming[0] - numDigits = serialNaming[1] - - serial = 0 - multiplier = 1 - for paramName in sorted(state.keys()): - if paramName in list(validParameters.keys()): - paramValue = state[paramName] - paramData = data[paramName] - paramNameMultiplier = len(paramData) - if paramValue in paramData: - paramValueIdx = paramData.index(paramValue) - serial += paramValueIdx * multiplier - multiplier *= paramNameMultiplier - name = "%s%0*u" % ("S" if isinstance(state, Solution) else "K", \ - numDigits, serial) - return name - ######################################## @ staticmethod @@ -4427,60 +4213,10 @@ def getParametersIndented(state, indent): s += "%s%s: %s\n" % (indent, str(key), str(state[key])) return s - ######################################## - @ staticmethod - @ lru_cache(maxsize=None) - def getParameterNameAbbreviation( name: str ): - return ''.join(c for c in name if c.isupper()) - - ######################################## class NonprimitiveParameterValueException(Exception): pass - @ staticmethod - @ lru_cache(maxsize=None) - def getPrimitiveParameterValueAbbreviation(key, value): - if isinstance(value, str): - return Solution.getParameterNameAbbreviation(value) - elif isinstance(value, bool): - return "1" if value else "0" - elif isinstance(value, int): - if value >= 0: - return "%u" % value - else: # -1 -> n1 - return "n%01u" % abs(value) - elif isinstance(value, ProblemType): - return str(value) - elif isinstance(value, float): - val1 = int(value) - val2 = int(round(value*100)) - int(value)*100 - if val2 > 0: - s = "%dp%s" % (val1,str(val2).zfill(2)) - else: - s = "%d" % (val1) - return s - - ######################################## - - @ staticmethod - def getParameterValueAbbreviation( key, value ): - if key == "ISA": - return f"{value[0]}{value[1]}{value[2]:x}" - - compositieTypes = (dict, list, tuple,) - - if not isinstance(value, compositieTypes): - return Solution.getPrimitiveParameterValueAbbreviation(key, value) - elif isinstance(value, tuple): - return ''.join(str(v) for v in value) - elif isinstance(value, list): - return '_'.join(Solution.getParameterValueAbbreviation(key, v) for v in value) - elif isinstance(value, dict): - return "_".join(f"{pos:d}{k:d}" for pos,k in value.items()) - else: - printExit('Parameter {key}={value} is new object type ({t})'.format(key=key, value=value, t=type(value))) - return str(value) ########################## # make class look like dict @@ -4502,7 +4238,7 @@ def __setitem__(self, key, value): def __str__(self): if self._name is None: - self._name = Solution.getNameFull(self._state, self.splitGSU) + self._name = getNameFull(self._state, self.splitGSU) return self._name def __repr__(self): diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index e1f2771678..c79663337c 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -36,6 +36,10 @@ CHeader, DebugConfig, DepthUConfig, + getKernelFileBase, + getKeyNoInternalArgs, + getMinNaming, + getSerialNaming, gfxToIsa, HR, IsaInfo, @@ -90,14 +94,14 @@ class KernelCodeGenResult(NamedTuple): wavefrontSize: int -def processKernelSource(kernelWriterAssembly, ti, useShortNames, kernel) -> KernelCodeGenResult: +def processKernelSource(kernelWriterAssembly, ti, useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) -> KernelCodeGenResult: """ Generate source for a single kernel. Returns (error, source, header, kernelName). """ kernelWriter = kernelWriterAssembly kernelWriter.setTensileInstructions(ti) - asmFilename = kernelWriter.getKernelFileBase(useShortNames, kernel) + asmFilename = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) err, src = kernelWriter.getSourceFileString(kernel, useShortNames) header = kernelWriter.getHeaderFileString(kernel) objFilename = kernel._state.get("codeObjectFile", None) @@ -207,6 +211,8 @@ def writeSolutionsAndKernels( kernelWriterAssembly, splitGSU: bool, cmdlineArchs: List[str], + kernelSerialNaming, + kernelMinNaming, errorTolerant=False, generateSourcesAndExit=False, compress=True, @@ -231,7 +237,7 @@ def writeSolutionsAndKernels( visited = set() duplicates = 0 for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(useShortNames, k) + base = getKernelFileBase(useShortNames, kernelSerialNaming, k) k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -242,7 +248,13 @@ def writeSolutionsAndKernels( numKernels = len(asmKernels) assert numKernels == numAsmKernels, "Only assembly kernels are supported in TensileLite" asmIter = zip( - itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), itertools.repeat(useShortNames), asmKernels + itertools.repeat(kernelWriterAssembly), + itertools.repeat(TensileInstructions()), + itertools.repeat(useShortNames), + itertools.repeat(splitGSU), + itertools.repeat(kernelMinNaming), + itertools.repeat(kernelSerialNaming), + asmKernels ) asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels", return_as="list") removeInvalidSolutionsAndKernels( @@ -272,9 +284,11 @@ def assemble(ret): asmToolchain.bundler, globalParameters["ROCmLdPath"], asmKernels, - kernelWriterAssembly, + kernelSerialNaming, + kernelMinNaming, destLibPath, assemblyTmpPath, + splitGSU, compress, useShortNames ) @@ -299,6 +313,8 @@ def writeSolutionsAndKernelsTCL( kernelHelperObjs, kernelWriterAssembly, cmdlineArchs: List[str], + kernelSerialNaming, + kernelMinNaming, compress=True, useShortNames=False, ): @@ -318,8 +334,9 @@ def writeSolutionsAndKernelsTCL( visited = set() duplicates = 0 + splitGSU = False for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(useShortNames, k) + base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -333,8 +350,15 @@ def assemble(ret): asmToolchain.assembler(isaToGfx(isa), wavefrontsize, str(p), str(p.with_suffix(".o"))) unaryProcessKernelSource = functools.partial( - processKernelSource, kernelWriterAssembly, TensileInstructions(), useShortNames + processKernelSource, + kernelWriterAssembly, + TensileInstructions(), + useShortNames, + splitGSU, + kernelMinNaming, + kernelSerialNaming ) + unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) ret = ParallelMap2( @@ -349,9 +373,11 @@ def assemble(ret): asmToolchain.bundler, globalParameters["ROCmLdPath"], asmKernels, - kernelWriterAssembly, + kernelSerialNaming, + kernelMinNaming, destLibPath, assemblyTmpPath, + splitGSU, compress, useShortNames ) @@ -371,25 +397,6 @@ def assemble(ret): return len(uniqueAsmKernels) -@timing -def getSolutionAndKernelWriters( - solutions, - kernels, - assembler: Assembler, -): - kernelSerialNaming = Solution.getSerialNaming(kernels) - solutionMinNaming = Solution.getMinNaming(solutions) - kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, - kernelSerialNaming, - assembler, - DebugConfig(), - ) - - return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) - - @timing def copyStaticFiles(outputPath): libraryStaticFiles = [ @@ -418,7 +425,7 @@ def generateKernelObjectsFromSolutions(solutions): for solution in solutions: solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel, splitGSU) + kName = getKeyNoInternalArgs(kernel, splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -637,8 +644,13 @@ def validLogicFile(p: Path): ) kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) - kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler + kernelSerialNaming = getSerialNaming(kernels) + kernelMinNaming = getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly( + kernelMinNaming, + kernelSerialNaming, + asmToolchain.assembler, + DebugConfig(), ) copyStaticFiles(outputPath) @@ -651,6 +663,8 @@ def validLogicFile(p: Path): kernelHelperObjs, kernelWriterAssembly, archs, + kernelSerialNaming, + kernelMinNaming, useShortNames=arguments["ShortNames"], compress=arguments["UseCompression"], ) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 04b14c1a4a..50ddfb265b 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import List, Union, NamedTuple -from ..Common import print2, ensurePath, isaToGfx +from ..Common import print2, ensurePath, isaToGfx, getKernelFileBase from ..KernelWriterAssembly import KernelWriterAssembly from ..SolutionStructs import Solution @@ -78,9 +78,11 @@ def buildAssemblyCodeObjectFiles( bundler: Bundler, ldPath: str, kernels: List[Solution], - writer: KernelWriterAssembly, + kernelSerialNaming, + kernelMinNaming, destDir: Union[Path, str], asmDir: Union[Path, str], + splitGSU: bool, compress: bool=True, useShortNames: bool=False, ): @@ -112,14 +114,14 @@ def buildAssemblyCodeObjectFiles( gfx = isaToGfx(arch) - objectFiles = [str(asmDir / (writer.getKernelFileBase(useShortNames, k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] + objectFiles = [str(asmDir / (getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] coFileMap = collections.defaultdict(list) if len(objectFiles): coFileMap[asmDir / ("TensileLibrary_"+ gfx + extCoRaw)] = objectFiles for kernel in archKernels: coName = kernel.get("codeObjectFile", None) if coName: - coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(useShortNames, kernel) + extObj))) + coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) + extObj))) for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) linker(objFiles, str(coFileRaw)) From 4382f49b23c067cb6f0a968250148630f2240f9f Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 17:13:04 +0000 Subject: [PATCH 067/105] Fix tox tests --- tensilelite/Tensile/BenchmarkProblems.py | 9 ++++++--- tensilelite/Tensile/LibraryLogic.py | 6 +++--- tensilelite/Tensile/TensileCreateLibrary/Run.py | 6 +++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index e04e12695b..2dd532b199 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -46,8 +46,9 @@ from Tensile.Toolchain.Component import Assembler from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ - BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, IsaVersion, isaToGfx, \ - DepthUConfig, getMinNaming, getNameMin, getSerialNaming, getNameFull + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, \ + DepthUConfig, getMinNaming, getNameMin, getSerialNaming, getNameFull, \ + getKeyNoInternalArgs def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ @@ -182,7 +183,7 @@ def writeBenchmarkFiles( for solution in tqdm(solutions, "Finding unique solutions"): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel, debugConfig.splitGSU) + kName = getKeyNoInternalArgs(kernel, debugConfig.splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -215,6 +216,8 @@ def writeBenchmarkFiles( kernelWriterAssembly, debugConfig.splitGSU, [isaToGfx(isa) for isa in isaInfoMap.keys()], + kernelSerialNaming, + kernelMinNaming, errorTolerant=True, generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], # put in debug config compress=False, diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 27dd5fb3bb..6eb2899089 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -26,8 +26,8 @@ from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig, getMinNaming, getNameMin -from .SolutionStructs import Solution + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig, \ + getMinNaming, getNameMin, getNameFull from . import LibraryIO from . import SolutionSelectionLibrary @@ -129,7 +129,7 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL s["SolutionIndex"] = i s["SolutionNameMin"] = getNameMin(s, solutionMinNaming, splitGSU) s["KernelNameMin"] = getNameMin(s, solutionMinNaming, splitGSU, True) - print1("(%2u) %s : %s" % (i, getNameMin(s, solutionMinNaming, splitGSU), Solution.getNameFull(s, splitGSU))) + print1("(%2u) %s : %s" % (i, getNameMin(s, solutionMinNaming, splitGSU), getNameFull(s, splitGSU))) if enableTileSelection: validSelectionSolutions = SolutionSelectionLibrary.analyzeSolutionSelection(problemType, selectionFileNameList, \ diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index c79663337c..a92223e8c7 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -129,7 +129,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, ) print(kernels[kernIdx]["SolutionNameMin"]) removeKernels.append(kernels[kernIdx]) - kName = Solution.getKeyNoInternalArgs(kernels[kernIdx], splitGSU) + kName = getKeyNoInternalArgs(kernels[kernIdx], splitGSU) if kName not in removeKernelNames: removeKernelNames.append(kName) removeResults.append(results[kernIdx]) @@ -147,7 +147,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, ): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel, splitGSU) + kName = getKeyNoInternalArgs(kernel, splitGSU) if kName in removeKernelNames: removeSolutions.append(solution) break @@ -237,7 +237,7 @@ def writeSolutionsAndKernels( visited = set() duplicates = 0 for k in asmKernels: - base = getKernelFileBase(useShortNames, kernelSerialNaming, k) + base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") From 7f6f16c8044bc6dac7495101eee41a7a01361003 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 17:45:05 +0000 Subject: [PATCH 068/105] Move minkforgsu to Module local constant --- tensilelite/Tensile/Contractions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 2db3e302ef..970bb4d08f 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -34,6 +34,7 @@ IsaInfo, DepthUConfig from Tensile.Toolchain.Component import Assembler +MIN_K_FOR_GSU = 32 @state_key_ordering class FreeIndex: StateKeys = ['isA', 'i', 'c', 'd'] @@ -515,7 +516,7 @@ def CompoundPredicates(cls, state, problemType): rv += [cls('BufferStoreOffsetLimitCheck', value=state['MacroTile1'])] if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None and not state["StreamK"]: - value = 32 # globalParameters['MinKForGSU'] + value = MIN_K_FOR_GSU rv += [cls('GlobalSplitUCheckMinK', value=[value, state["GlobalSplitU"]])] if ('WorkGroupMappingXCC' in state) and ('WorkGroupMappingXCCGroup' in state): From 6f57df5430bc7b9b9ac99243a4ffe18a90912e2b Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 17:51:44 +0000 Subject: [PATCH 069/105] missing file --- tensilelite/Tensile/Common/Naming.py | 280 +++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 tensilelite/Tensile/Common/Naming.py diff --git a/tensilelite/Tensile/Common/Naming.py b/tensilelite/Tensile/Common/Naming.py new file mode 100644 index 0000000000..d5b8cba042 --- /dev/null +++ b/tensilelite/Tensile/Common/Naming.py @@ -0,0 +1,280 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ +from copy import deepcopy +from functools import lru_cache +from .Constants import MAX_FILENAME_LENGTH +from .ValidParameters import validParameters + +######################################## +# create a dictionary with booleans on whether to include parameter in name +def getMinNaming(objs): + nonCKObjs = [obj for obj in objs if not ("CustomKernelName" in obj and obj["CustomKernelName"])] + # early return + if len(nonCKObjs) == 0: + return {} + # determine keys + requiredParameters = {} + if hasattr(nonCKObjs[0], "_state"): + keys = list(nonCKObjs[0]._state.keys()) + else: + keys = list(nonCKObjs[0].keys()) + # only 1, rather than name being nothing, it'll be everything + if len(nonCKObjs) == 1: + for key in keys: + if key in list(validParameters.keys()): + requiredParameters[key] = False + else: + for key in keys: + required = False + if key in list(validParameters.keys()): + for i in range(1, len(nonCKObjs)): + if nonCKObjs[0][key] != nonCKObjs[i][key]: + required = True + break + if required: + requiredParameters[key] = True + else: + requiredParameters[key] = False + requiredParameters["GlobalSplitU"] = True + requiredParameters["WorkGroupMapping"] = True + if "MatrixInstM" in nonCKObjs[0]._state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + requiredParameters["ProblemType"] = False # always prepended + requiredParameters["MacroTile0"] = False # always prepended + requiredParameters["MacroTile1"] = False # always prepended + requiredParameters["DepthU"] = False # always prepended + requiredParameters["MatrixInstruction"] = False # always prepended + requiredParameters["MatrixInstM"] = False # always prepended + requiredParameters["MatrixInstN"] = False # always prepended + requiredParameters["MatrixInstK"] = False # always prepended + requiredParameters["MatrixInstB"] = False # always prepended + requiredParameters["MatrixInstBM"] = False # always prepended + requiredParameters["MatrixInstBN"] = False # always prepended + requiredParameters["CustomKernelName"] = False # Will not affect naming + requiredParameters["Kernel"] = True # distinguish kernels from solutions + # for single-source compilation + return requiredParameters + + +def getKeyNoInternalArgs(state, splitGSU: bool): + state_copy = deepcopy(state) + state_copy["ProblemType"]["GroupedGemm"] = False + if splitGSU: + state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] + elif state["GlobalSplitU"] > 0: + state_copy["GlobalSplitU"] = "M" + state_copy["WorkGroupMapping"] = "M" + state_copy["WorkGroupMappingXCC"] = "M" + state_copy["WorkGroupMappingXCCGroup"] = "M" + state_copy["StaggerU"] = "M" + state_copy["StaggerUStride"] = "M" + state_copy["StaggerUMapping"] = "M" + state_copy["GlobalSplitUCoalesced"] = "M" + state_copy["GlobalSplitUWorkGroupMappingRoundRobin"] = "M" + return state_copy + + +def getNameFull(state, splitGSU: bool): + requiredParameters = {} + for key in state: + if key in list(validParameters.keys()): + requiredParameters[key] = True + if "MatrixInstM" in state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + return getNameMin(state, requiredParameters, splitGSU) + + +@lru_cache(maxsize=None) +def getParameterNameAbbreviation( name: str ): + return ''.join(c for c in name if c.isupper()) + +@ lru_cache(maxsize=None) +def getPrimitiveParameterValueAbbreviation(key, value): + if isinstance(value, str): + return getParameterNameAbbreviation(value) + elif isinstance(value, bool): + return "1" if value else "0" + elif isinstance(value, int): + if value >= 0: + return "%u" % value + else: # -1 -> n1 + return "n%01u" % abs(value) + elif isinstance(value, ProblemType): + return str(value) + elif isinstance(value, float): + val1 = int(value) + val2 = int(round(value*100)) - int(value)*100 + if val2 > 0: + s = "%dp%s" % (val1,str(val2).zfill(2)) + else: + s = "%d" % (val1) + return s + + +def getParameterValueAbbreviation(key, value): + if key == "ISA": + return f"{value[0]}{value[1]}{value[2]:x}" + compositieTypes = (dict, list, tuple,) + if not isinstance(value, compositieTypes): + return getPrimitiveParameterValueAbbreviation(key, value) + elif isinstance(value, tuple): + return ''.join(str(v) for v in value) + elif isinstance(value, list): + return '_'.join(getParameterValueAbbreviation(key, v) for v in value) + elif isinstance(value, dict): + return "_".join(f"{pos:d}{k:d}" for pos,k in value.items()) + else: + raise Exception(f"Parameter {key}={value} is new object type ({type(value)})") + + +def getNameMin(state, requiredParameters, splitGSU: bool, ignoreInternalArgs = False): + if "CustomKernelName" in state and state["CustomKernelName"]: + return state["CustomKernelName"] + + components = [] + backup = state["ProblemType"]["GroupedGemm"] + if ignoreInternalArgs: + state["ProblemType"]["GroupedGemm"] = False + if "ProblemType" in state: + components.append(f'{str(state["ProblemType"])}') + # name += str(state["ProblemType"]) + "_" + if ignoreInternalArgs: + state["ProblemType"]["GroupedGemm"] = backup + if "MacroTile0" in state \ + and "MacroTile1" in state \ + and "DepthU" in state: + components.append(f'{getParameterNameAbbreviation("MacroTile")}{state["MacroTile0"]}x{state["MacroTile1"]}x{state["DepthU"]}') + if "MatrixInstM" in state: + components.append(f'{getParameterNameAbbreviation("MatrixInstruction")}{state["MatrixInstM"]}x{state["MatrixInstN"]}x{state["MatrixInstB"]}') + backup = state["GlobalSplitU"] + if ignoreInternalArgs: + if splitGSU: + state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] + elif state["GlobalSplitU"] > 0: + requiredParameters["GlobalSplitU"] = False + requiredParameters["WorkGroupMapping"] = False + requiredParameters["WorkGroupMappingXCC"] = False + requiredParameters["WorkGroupMappingXCCGroup"] = False + requiredParameters["StaggerU"] = False + requiredParameters["StaggerUStride"] = False + requiredParameters["StaggerUMapping"] = False + requiredParameters["GlobalSplitUCoalesced"] = False + requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = False + useWaveTile, useThreadTile = requiredParameters.get("MIWaveTile", False), requiredParameters.get("ThreadTile", False) + if 'MatrixInstM' in state: + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + else: + requiredParameters["MIWaveTile"] = False + requiredParameters["ThreadTile"] = True + components.append('SN') + for key in sorted(state.keys()): + if key in requiredParameters and key[0] != '_': + if requiredParameters[key] and key != "CustomKernelName": + components.append(f'{getParameterNameAbbreviation(key)}{getParameterValueAbbreviation(key, state[key])}') + state["GlobalSplitU"] = backup + requiredParameters["GlobalSplitU"] = True + requiredParameters["WorkGroupMapping"] = True + requiredParameters["WorkGroupMappingXCC"] = True + requiredParameters["WorkGroupMappingXCCGroup"] = True + requiredParameters["StaggerU"] = True + requiredParameters["StaggerUStride"] = True + requiredParameters["StaggerUMapping"] = True + requiredParameters["GlobalSplitUCoalesced"] = True + requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = True + requiredParameters["MIWaveTile"] = useWaveTile + requiredParameters["ThreadTile"] = useThreadTile + return '_'.join(components) + + +def getSerialNaming(objs): + data = {} + for obj in objs: + for paramName in sorted(obj.keys()): + if paramName in validParameters.keys(): + paramValue = obj[paramName] + if paramName in data: + if paramValue not in data[paramName]: + data[paramName].append(paramValue) + else: + data[paramName] = [ paramValue ] + maxObjs = 1 + for paramName in data: + if not isinstance(data[paramName][0], dict): + data[paramName] = sorted(data[paramName]) + maxObjs *= len(data[paramName]) + numDigits = len(str(maxObjs)) + return [ data, numDigits ] + + +def getNameSerial(state, serialNaming): + data = serialNaming[0] + numDigits = serialNaming[1] + serial = 0 + multiplier = 1 + for paramName in sorted(state.keys()): + if paramName in list(validParameters.keys()): + paramValue = state[paramName] + paramData = data[paramName] + paramNameMultiplier = len(paramData) + if paramValue in paramData: + paramValueIdx = paramData.index(paramValue) + serial += paramValueIdx * multiplier + multiplier *= paramNameMultiplier + name = "%s%0*u" % ("S" if hasattr(state, "_state") else "K", \ + numDigits, serial) + return name + + +def shortenFileBase(kernelMinNaming, splitGSU, kernel): + base = getKernelName(kernelMinNaming, splitGSU, kernel) + if len(base) <= MAX_FILENAME_LENGTH: + return base + import hashlib + import base64 + pivot = MAX_FILENAME_LENGTH * 3 // 4 + firstPart = base[:pivot] + secondPart = base[pivot:] + secondHash = hashlib.sha256(secondPart.encode()).digest() + secondPart = base64.b64encode(secondHash, b'_-').decode() + return firstPart + secondPart + + +def getKernelFileBase(useShortNames: bool, splitGSU: bool, kernelMinNaming, kernelSerialNaming, kernel): + if "CustomKernelName" in kernel and kernel["CustomKernelName"]: + fileBase = kernel["CustomKernelName"] + elif useShortNames: + fileBase = getNameSerial(kernel, kernelSerialNaming) + else: + fileBase = shortenFileBase(kernelMinNaming, splitGSU, kernel) + return fileBase + + +def getKernelName(kernelMinNaming, splitGSU, kernel): + kernelName = getNameMin(kernel, kernelMinNaming, splitGSU, True) + return kernelName From 371011e72197fc1165ebb3ba31398d7e854a422a Mon Sep 17 00:00:00 2001 From: David Dixon Date: Mon, 17 Feb 2025 21:00:34 +0000 Subject: [PATCH 070/105] Only compute kernel base name once --- .../Tensile/TensileCreateLibrary/Run.py | 23 ++++++++----------- tensilelite/Tensile/Toolchain/Assembly.py | 15 ++++-------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index a92223e8c7..86d8edc580 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -239,6 +239,8 @@ def writeSolutionsAndKernels( for k in asmKernels: base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) k.duplicate = True if base in visited else False + if not k.duplicate: + k["BaseName"] = base duplicates += k.duplicate print2(f"Duplicate: {base}") visited.add(base) @@ -284,13 +286,9 @@ def assemble(ret): asmToolchain.bundler, globalParameters["ROCmLdPath"], asmKernels, - kernelSerialNaming, - kernelMinNaming, destLibPath, assemblyTmpPath, - splitGSU, compress, - useShortNames ) buildSourceCodeObjectFiles( srcToolchain.compiler, @@ -337,6 +335,7 @@ def writeSolutionsAndKernelsTCL( splitGSU = False for k in asmKernels: base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) + k["BaseName"] = base k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -372,14 +371,10 @@ def assemble(ret): asmToolchain.linker, asmToolchain.bundler, globalParameters["ROCmLdPath"], - asmKernels, - kernelSerialNaming, - kernelMinNaming, + uniqueAsmKernels, destLibPath, assemblyTmpPath, - splitGSU, compress, - useShortNames ) writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) @@ -387,11 +382,11 @@ def assemble(ret): buildSourceCodeObjectFiles( srcToolchain.compiler, srcToolchain.bundler, - destLibPath, - objectTmpPath, - outputPath, - srcKernelFile, - cmdlineArchs, + destLibPath, + objectTmpPath, + outputPath, + srcKernelFile, + cmdlineArchs, ) return len(uniqueAsmKernels) diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 50ddfb265b..831c28b7c6 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -30,8 +30,7 @@ from pathlib import Path from typing import List, Union, NamedTuple -from ..Common import print2, ensurePath, isaToGfx, getKernelFileBase -from ..KernelWriterAssembly import KernelWriterAssembly +from ..Common import print2, isaToGfx from ..SolutionStructs import Solution from .Component import Assembler, Linker, Bundler @@ -78,13 +77,9 @@ def buildAssemblyCodeObjectFiles( bundler: Bundler, ldPath: str, kernels: List[Solution], - kernelSerialNaming, - kernelMinNaming, destDir: Union[Path, str], asmDir: Union[Path, str], - splitGSU: bool, compress: bool=True, - useShortNames: bool=False, ): """Builds code object files from assembly files @@ -97,14 +92,12 @@ def buildAssemblyCodeObjectFiles( compress: Whether to compress the code object files. """ - isAsm = lambda k: k["KernelLanguage"] == "Assembly" - extObj = ".o" extCo = ".co" extCoRaw = ".co.raw" archKernelMap = collections.defaultdict(list) - for k in filter(isAsm, kernels): + for k in kernels: archKernelMap[tuple(k['ISA'])].append(k) coFiles = [] @@ -114,14 +107,14 @@ def buildAssemblyCodeObjectFiles( gfx = isaToGfx(arch) - objectFiles = [str(asmDir / (getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] + objectFiles = [str(asmDir / (k["BaseName"] + extObj)) for k in archKernels if 'codeObjectFile' not in k] coFileMap = collections.defaultdict(list) if len(objectFiles): coFileMap[asmDir / ("TensileLibrary_"+ gfx + extCoRaw)] = objectFiles for kernel in archKernels: coName = kernel.get("codeObjectFile", None) if coName: - coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) + extObj))) + coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (kernel["BaseName"] + extObj))) for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) linker(objFiles, str(coFileRaw)) From 029a0b4bc27448182c8ce249c25dc0311e877851 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 18 Feb 2025 16:01:28 +0000 Subject: [PATCH 071/105] Factor out ProblemType to avoid circular import --- tensilelite/Tensile/Activation.py | 2 +- tensilelite/Tensile/BenchmarkProblems.py | 3 +- tensilelite/Tensile/BenchmarkStructs.py | 3 +- tensilelite/Tensile/Common/Architectures.py | 2 +- .../Tensile/Common/GlobalParameters.py | 100 +-- tensilelite/Tensile/Common/Naming.py | 4 +- tensilelite/Tensile/Common/Utilities.py | 9 + tensilelite/Tensile/Common/ValidParameters.py | 2 +- tensilelite/Tensile/Contractions.py | 2 +- tensilelite/Tensile/LibraryIO.py | 3 +- tensilelite/Tensile/ProblemType.py | 712 ++++++++++++++++++ tensilelite/Tensile/SolutionStructs.py | 591 +-------------- tensilelite/Tensile/TensileClientConfig.py | 3 +- .../Tensile/TensileCreateLibrary/Run.py | 24 +- 14 files changed, 748 insertions(+), 712 deletions(-) create mode 100644 tensilelite/Tensile/ProblemType.py diff --git a/tensilelite/Tensile/Activation.py b/tensilelite/Tensile/Activation.py index 67f0f6d2c8..6b92561af6 100644 --- a/tensilelite/Tensile/Activation.py +++ b/tensilelite/Tensile/Activation.py @@ -31,7 +31,7 @@ TensileInstructions from .TensileInstructions.Enums import * from .TensileInstructions.Instructions import * -from .Common import printExit, printWarning +from Tensile.Common.Utilities import printExit, printWarning from dataclasses import dataclass, field diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 2dd532b199..0bba1b949d 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -38,7 +38,8 @@ from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni from .KernelWriterAssembly import KernelWriterAssembly from .KernelWriter import DebugConfig -from .SolutionStructs import Solution, ProblemType, ProblemSizes +from .SolutionStructs import Solution, ProblemSizes +from Tensile.ProblemType import ProblemType from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index b6f5d84e23..0d2b7cf76c 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -28,8 +28,9 @@ defaultBenchmarkCommonParameters, validParameters, globalParameters, \ defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes from .CustomKernels import getAllCustomKernelNames -from .SolutionStructs import ProblemType, ProblemSizes, ActivationArgs, BiasTypeArgs, \ +from .SolutionStructs import ProblemSizes, ActivationArgs, BiasTypeArgs, \ FactorDimArgs +from Tensile.ProblemType import ProblemType def getDefaultsForMissingParameters(paramList, defaultParams): diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index 0897e8cdd9..9f885461cb 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -56,7 +56,6 @@ "gfx1201": "gfx1201", } - SUPPORTED_ISA = [ IsaVersion(8, 0, 3), IsaVersion(9, 0, 0), @@ -77,6 +76,7 @@ IsaVersion(12, 0, 1), ] +SUPPORTED_GFX = [isaToGfx(isa) for isa in SUPPORTED_ISA] def gfxToIsa(name: str) -> Optional[IsaVersion]: """Extracts the ISA version from a given gfx architecture name. diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 090e69dbcd..9122e7b83c 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -438,97 +438,7 @@ defaultSolution[key] = value[0] # other non-benchmark options for solutions -################################################################################ -# Default Problem Type -################################################################################ -defaultProblemType = { - # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users - # =TensorContraction requires specifying - "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights - "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. - # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) - "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter - "UseE": False, # =True use output E to output gemm results before activation - "Gradient": False, # =True set globalWriteElements to gradient mode - "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction - "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. - "UseScaleAB": "", # Support "", "Scalar", and "Vector" - "UseScaleCD": False, # =True use scaleC, scaleD - "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction - "HighPrecisionAccumulate": False, # f32 += f16*f16 - "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. - "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case - "ComplexConjugateB": False, - "StochasticRounding": False, # By default, IEEE RNE rounding - # for OperationType == GEMM - "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" - "TransposeB": True, - "Batched": False, # add batching dimension - "StridedBatched": True, # use to select general batch or strided batch - "GroupedGemm": False, # use to select general batch or strided batch - # for OperationType == TensorContraction - # - Indices < NumIndicesC are Free or Batch indices and appear in C and D - # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. - # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. - # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. - # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. - # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. - # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. - # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. - # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. - "IndexAssignmentsA": [0, 2], - "IndexAssignmentsB": [1, 2], - "NumIndicesC": 2, - # use initial strides for AB. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) - "UseInitialStridesAB": False, - # use initial strides for CD. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - Additional multiply on the store address path - # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. - "UseInitialStridesCD": False, - "AllowNoFreeDims": False, # allow A or B to specify no free dims - # (if false, A and B must have at least one free dim) - # (if true, A and B must have at least one free or batch dim) - # SetConstStride* sets the specified stride in the problem. - # These no longer generate predicates - see AssertStrideEqualA/B below - # List of pairs of [index, constValue]. - # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) - # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets - # strideA for index3 to constant '1' and stride for index2 to constant '4'. - "SetConstStrideA": [], - "SetConstStrideB": [], - "SetConstStrideBias": [], - # Summation dimension indices - "MirrorDimsA": [], - "MirrorDimsB": [], - "MirrorDimsMetadata": [], - # for LD description - "NumIndicesLD": 4, - "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB - # Tile aware solution selection - "TileAwareSelection": False, - # Activation - "Activation": False, - "ActivationNoGuard": False, - # AmaxD - "OutputAmaxD": False, - # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. - "SupportUserArgs": True, - "SwizzleTensorA": False, - "SwizzleTensorB": False, -} + defaultProblemSizes = [{"Range": [[2880], 0, 0]}] defaultBenchmarkFinalProblemSizes = [{"Range": [[64, 64, 64, 512], 0, 0]}] @@ -753,12 +663,4 @@ def restoreClocks(): atexit.register(restoreClocks) - setupRestoreClocks() - - -def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): - if key in sourceDictionary: - destinationDictionary[key] = deepcopy(sourceDictionary[key]) - else: - destinationDictionary[key] = deepcopy(defaultDictionary[key]) diff --git a/tensilelite/Tensile/Common/Naming.py b/tensilelite/Tensile/Common/Naming.py index d5b8cba042..1da89b4751 100644 --- a/tensilelite/Tensile/Common/Naming.py +++ b/tensilelite/Tensile/Common/Naming.py @@ -26,6 +26,8 @@ from .Constants import MAX_FILENAME_LENGTH from .ValidParameters import validParameters +from Tensile.ProblemType import ProblemType + ######################################## # create a dictionary with booleans on whether to include parameter in name def getMinNaming(objs): @@ -124,7 +126,7 @@ def getPrimitiveParameterValueAbbreviation(key, value): return "%u" % value else: # -1 -> n1 return "n%01u" % abs(value) - elif isinstance(value, ProblemType): + elif isinstance(value, ProblemType): # will need to deal with this return str(value) elif isinstance(value, float): val1 = int(value) diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 39814c71d2..1746b0f5f0 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -27,6 +27,8 @@ import os import sys import time + +from copy import deepcopy from enum import Enum from .Types import IsaInfo @@ -285,3 +287,10 @@ def ClientExecutionLock(lockPath: str): import filelock return filelock.FileLock(lockPath) + + +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index 571a2c6c40..5d0a61d636 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -748,7 +748,7 @@ # if source, Runtime will determine language # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm "KernelLanguage": ["Assembly"], - # We set ISA in multiple places + # We set validParams["ISA"] in multiple places "ISA": validISA, # arch for assembly kernels # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 970bb4d08f..e9d3a293d8 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -28,10 +28,10 @@ from .TensileInstructions import DataType from . import Hardware from . import Properties -from .SolutionStructs import getBiasDataTypeListDefault from .SolutionStructs import Solution as OriginalSolution from .Common import gfxToIsa, internalParameters, state, state_key_ordering, \ IsaInfo, DepthUConfig +from Tensile.ProblemType import getBiasDataTypeListDefault from Tensile.Toolchain.Component import Assembler MIN_K_FOR_GSU = 32 diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 959502b9e0..4b84fa1a09 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -23,7 +23,8 @@ ################################################################################ from .CustomKernels import getCustomKernelConfig -from .SolutionStructs import Solution, ProblemSizes, ProblemType +from .SolutionStructs import Solution, ProblemSizes +from Tensile.ProblemType import ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream from .Common import gfxToIsa, printExit, printWarning, print2, \ diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py new file mode 100644 index 0000000000..2787d0195f --- /dev/null +++ b/tensilelite/Tensile/ProblemType.py @@ -0,0 +1,712 @@ +################################################################################ +# +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from collections.abc import Mapping +from typing import List + +from Tensile.Activation import ActivationType +from Tensile.TensileInstructions.DataType import DataType +from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2 + +################################################################################ +# ProblemType +# name of solution should begin with name of problemType, and arguments can be listed out explicitly + +################################################################################ +# Default Problem Type +################################################################################ +_defaultProblemType = { + # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users + # =TensorContraction requires specifying + "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. + # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) + "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter + "UseE": False, # =True use output E to output gemm results before activation + "Gradient": False, # =True set globalWriteElements to gradient mode + "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction + "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. + "UseScaleAB": "", # Support "", "Scalar", and "Vector" + "UseScaleCD": False, # =True use scaleC, scaleD + "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction + "HighPrecisionAccumulate": False, # f32 += f16*f16 + "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. + "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case + "ComplexConjugateB": False, + "StochasticRounding": False, # By default, IEEE RNE rounding + # for OperationType == GEMM + "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" + "TransposeB": True, + "Batched": False, # add batching dimension + "StridedBatched": True, # use to select general batch or strided batch + "GroupedGemm": False, # use to select general batch or strided batch + # for OperationType == TensorContraction + # - Indices < NumIndicesC are Free or Batch indices and appear in C and D + # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. + # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. + # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. + # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. + # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. + # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. + # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. + # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. + "IndexAssignmentsA": [0, 2], + "IndexAssignmentsB": [1, 2], + "NumIndicesC": 2, + # use initial strides for AB. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) + "UseInitialStridesAB": False, + # use initial strides for CD. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - Additional multiply on the store address path + # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. + "UseInitialStridesCD": False, + "AllowNoFreeDims": False, # allow A or B to specify no free dims + # (if false, A and B must have at least one free dim) + # (if true, A and B must have at least one free or batch dim) + # SetConstStride* sets the specified stride in the problem. + # These no longer generate predicates - see AssertStrideEqualA/B below + # List of pairs of [index, constValue]. + # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) + # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets + # strideA for index3 to constant '1' and stride for index2 to constant '4'. + "SetConstStrideA": [], + "SetConstStrideB": [], + "SetConstStrideBias": [], + # Summation dimension indices + "MirrorDimsA": [], + "MirrorDimsB": [], + "MirrorDimsMetadata": [], + # for LD description + "NumIndicesLD": 4, + "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB + # Tile aware solution selection + "TileAwareSelection": False, + # Activation + "Activation": False, + "ActivationNoGuard": False, + # AmaxD + "OutputAmaxD": False, + # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. + "SupportUserArgs": True, + "SwizzleTensorA": False, + "SwizzleTensorB": False, +} + +# The supported typed GEMM, each entry is (Ti, To, Tc). +# DataType (Ti) = The data-type of the input matrices: A/B +# DestDataType (To) = The data-type of the output matrices: C/D +# ComputeDataType (Tc) = The data-type of computation: alpha/beta: +# Cinternal: basically should == ComputeDataType +# This is used in _checkIfSupportedGEMMType() +_validGEMMTypes = [ + ("H", "H", "H"), + ("S", "S", "S"), + ("D", "D", "D"), + ("C", "C", "C"), + ("Z", "Z", "Z"), + ("H", "H", "S"), + ("H", "S", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I8", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("B8", "B", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), + ("B8F8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), # F8 NANOO + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("B8N", "B", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), + ("B8F8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + +class ProblemType(Mapping): + ######################################## + + @classmethod + def FromDefaultConfig(printIndexAssignmentInfo: bool): + return ProblemType(_defaultProblemType, printIndexAssignmentInfo) + + def __init__(self, config, printIndexAssignmentInfo: bool): + self.state = {} + + for key in _defaultProblemType: + assignParameterWithDefault(self.state, key, config, _defaultProblemType) + + # adjusting all data types + if "DataType" in config: + self["DataType"] = DataType(config["DataType"]) + self["DataTypeA"] = self["DataType"] + self["DataTypeB"] = self["DataType"] + else: + raise Exception("NO data type specified") + self["DataType"] = DataType(0) + self["DataTypeA"] = DataType(0) + self["DataTypeB"] = DataType(0) + + if "DataTypeA" in config: + self["DataTypeA"] = DataType(config["DataTypeA"]) + + if "DataTypeB" in config: + self["DataTypeB"] = DataType(config["DataTypeB"]) + + if "DestDataType" in config: + self["DestDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["DestDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO dest data type or data type specified") + self["DataType"] = DataType(0) + + self["DataTypeE"] = self["DestDataType"] + if "DataTypeE" in config: + self["DataTypeE"] = DataType(config["DataTypeE"]) + + if "ComputeDataType" in config: + self["ComputeDataType"] = DataType(config["ComputeDataType"]) + else: + if "DestDataType" in config: + self["ComputeDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["ComputeDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO compute data type, or dest data type, or data type specified") + self["DataType"] = DataType(0) + + # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. + # So far we don't have to set it in config yamls + self["DataTypeAmaxD"] = self["ComputeDataType"] + if "DataTypeAmaxD" in config: + self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) + + if self["Sparse"]: + self["DataTypeMetadata"] = DataType("I8") + + if "F32XdlMathOp" in config: + self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) + else: + self["F32XdlMathOp"] = DataType(0) + + # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. + if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: + print2("DataType == i8 and HPA == True; setting compute data type to int32") + self["ComputeDataType"] = DataType('i') + + if self["OperationType"] == "GEMM": + self._checkIfSupportedGEMMType() + self.initGEMM() + else: + raise Exception("Unsupported OperationType = %s" % self["OperationType"]) + + self.state["AssignedDerivedParameters"] = False + ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) + + for tc in ('A', 'B'): + for sc in self["SetConstStride%s"%tc] : + (anchorDim, stride) = sc[:2] + if anchorDim not in self.state["IndexAssignments%s"%tc]: + raise Exception("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) + + # Bias + # If compute data type is not equal to dest data type, tensile will run conversion kernel. + # In this case we don't need to apply bias in beta only kernel. + if "UseBias" in config: + if self["ComputeDataType"] != self["DestDataType"]: + self["BetaOnlyUseBias"] = False + else: + self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False + if "BiasDataTypeList" in config: + self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] + self["BiasDataTypeList"].sort() # Make name unique + else: + self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) + else: + self["BetaOnlyUseBias"] = False + self["BiasDataTypeList"] = [] + + # Activation + # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. + # Otherwise, ActivationType will be set to 'none'. + if "Activation" in config: + typeStr = config.get("ActivationType", 'none') + if typeStr not in ['all', 'hipblaslt_all']: + typeStr = 'none' + else: + typeStr = 'none' + self["ActivationType"] = ActivationType(typeStr) + if "ActivationComputeDataType" in config: + self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) + else: + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if self["ActivationType"] != 'none': + # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type + if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: + printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ + ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ + self["ComputeDataType"], \ + self["DestDataType"])) + self["ActivationComputeDataType"] = self["ComputeDataType"] + if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ + (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): + printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ + ActivationComputeDataType will be set to ComputeDataType automatically.") + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if "UseE" in config: + if config["UseE"]: + if self["ActivationType"] == 'none': + printWarning("Use E is disabled cause Activation is set to False.") + self["UseE"] = False + else: + self["UseE"] = config["UseE"] + else: + self["UseE"] = config["UseE"] + + if "Gradient" in config: + if config["Gradient"]: + if (not self["UseBias"]) and self["ActivationType"] == 'none': + printWarning("Gradient is disabled cause bias and activation are both disabled.") + self["Gradient"] = False + if self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is enabled cause Activation is enabled.") + self["UseE"] = True + elif self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is disabled cause Activation is disabled.") + self["UseE"] = False + # if self["UseScaleAlphaVec"]: + # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") + # self["UseScaleAlphaVec"] = False + self["Gradient"] = config["Gradient"] + + # Need gradient info + biasSrcList = ["A", "B", "D"] + if "BiasSrc" in config: + if not self["Gradient"] and config["BiasSrc"] != "D": + printWarning("BiasSrc is set to D cause Gradient is disabled.") + self["BiasSrc"] = "D" + elif self["Gradient"]: + # # Currently only supports D :) + # if config["BiasSrc"] != "D": + # raise Exception("BiasSrc currently only supports D.") + if config["BiasSrc"] not in biasSrcList: + raise Exception("BiasSrc only supports A, B, D.") + + if "ActivationNoGuard" in config: + self["ActivationNoGuard"] = config["ActivationNoGuard"] + if self["ActivationNoGuard"]: + if self["ActivationType"] == 'none': + printWarning("ActivationNoGuard is set to False cause Acivation is off.") + self["ActivationNoGuard"] = False + if (not self["Gradient"]): + printWarning("ActivationNoGuard is set to False cause Gradient is off.") + self["ActivationNoGuard"] = False + + ################################################################################ + # Function checkIfSupportedGEMMType: + # Assures 3 data-types are valid, supported and well-assigned + # See the discussion in ValidParameters.py for validGEMMTypes + ################################################################################ + def _checkIfSupportedGEMMType(self): + inType = self["DataType"] + outType = self["DestDataType"] + computeType = self["ComputeDataType"] + + gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) + if gemmType not in _validGEMMTypes: + raise Exception("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0], gemmType[1], gemmType[2])) + + ######################################## + def initGEMM(self): + sumIdx = 3 if self["Batched"] else 2 + self["IndexAssignmentsA"] = [0, sumIdx] # N + self["IndexAssignmentsB"] = [sumIdx, 1] # N + if self.state["Sparse"] == 2: + self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) + else: + self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) + if self["TransposeA"]: + self["IndexAssignmentsA"] = [sumIdx, 0] # T + if self["TransposeB"]: + self["IndexAssignmentsB"] = [1, sumIdx] # T + if self["Batched"]: + self["IndexAssignmentsA"].append(2) + self["IndexAssignmentsB"].append(2) + self["IndexAssignmentsMetadata"].append(2) + self["NumIndicesC"] = 3 + else: + self["NumIndicesC"] = 2 + + self["NumIndicesLD"] = 4 + self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 + for i in range(1, len(self["IndexAssignmentsLD"])): + self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 + + ######################################## + def isGEMM(self): + return self.operationType == 0 + + ######################################## + # determine d0, d1, dU + @staticmethod + def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): + if "AssignedDerivedParameters" in state: + if state["AssignedDerivedParameters"]: + return + state["AssignedDerivedParameters"] = False + + state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ + max(state["IndexAssignmentsB"])+1) + + # determine num free, batch + state["IndicesFree"] = [] + state["IndicesBatch"] = [] + state["IndicesSummation"] = [] + + for i in range(0, state["NumIndicesC"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesBatch"].append(i) + + elif inA or inB: + state["IndicesFree"].append(i) + else: + raise Exception("invalid index %u (inC but not (inA or inB))" % i) + + # determine num summation + for i in range(state["NumIndicesC"], state["TotalIndices"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesSummation"].append(i) + else: + raise Exception("invalid index %u (expected summation but not (inA and inB))" % i) + # print index assignments + if printIndexAssignmentInfo: + print("IndicesFree: %s" % state["IndicesFree"]) + print("IndicesBatch: %s" % state["IndicesBatch"]) + print("IndicesSum: %s" % state["IndicesSummation"]) + print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) + print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) + print("NumIndicesC: %s" % state["NumIndicesC"]) + + for k in ('IndexAssignmentsA','IndexAssignmentsB'): + if len(state[k]) != len(set(state[k])): + raise Exception("duplicate index in %s=%s"% (k,state[k])) + + state["NumIndicesFree"] = len(state["IndicesFree"]) + state["NumIndicesBatch"] = len(state["IndicesBatch"]) + state["NumIndicesSummation"] = len(state["IndicesSummation"]) + if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : + raise Exception("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) + + # by default, unroll index will be the last/inner summation index + state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] + for i in range(0, len(state["IndexAssignmentsA"])): + if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: + state["IndexUnrollA"] = i + break + for i in range(0, len(state["IndexAssignmentsB"])): + if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: + state["IndexUnrollB"] = i + break + for i in range(0, len(state["IndexAssignmentsMetadata"])): + if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: + state["IndexUnrollM"] = i + break + #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) + #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) + + # assign d0, d1 + if state["AllowNoFreeDims"]: + dimList = state["IndicesFree"] + state["IndicesBatch"] + else: + dimList = state["IndicesFree"] + state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] + state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] + #print2("Index01A: %u" % state["Index01A"]) + #print2("Index01B: %u" % state["Index01B"]) + # Store code is optimized for 0 as the fastest-moving in memory + # whichever has lower stride in C (lower value), is 0, other is 1 + if state["Index01A"] < state["Index01B"]: + state["Index0"] = state["Index01A"] + state["Index1"] = state["Index01B"] + state["Tensor0"] = 0 + state["Tensor1"] = 1 + state["TileA"] = 0 + state["TileB"] = 1 + else: + state["Index0"] = state["Index01B"] + state["Index1"] = state["Index01A"] + state["Tensor0"] = 1 + state["Tensor1"] = 0 + state["TileA"] = 1 + state["TileB"] = 0 + + # generalize transpose + strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) + strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) + unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) + unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) + state["TLUA"] = strideIdxA < unrollIdxA + state["TLUB"] = strideIdxB < unrollIdxB + #state["TLUB"] = True # hack + + if printIndexAssignmentInfo: + print("TLUA: %s (stridePosA(%d) 1 : + name += "_FD%s"%("N" if factorDim == 2 else "MN") + + if self["UseE"]: + if self["Gradient"]: + name += "_Grad%s"%self["DataTypeE"].toChar() + else: + name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types + if self["OutputAmaxD"]: + name += "_AmaxD" + if self["Sparse"]: + if self["Sparse"] == 2: + name += "_SPB" + else: + name += "_SPA" + + # precision and other + # name += "_SB" if self["StridedBatched"] else "_GB" + if self["GroupedGemm"]: + name += "_GG" + else: + name += "" if self["StridedBatched"] else "_GB" # legacy + + # Activation Naming + if self["ActivationType"] != 'none': + if self["ActivationType"] == 'all': + name += "_A" + elif self["ActivationType"] == 'hipblaslt_all': + name += "_HA" + else: + name += "_%s"%str(self["ActivationType"]).upper() + name += self["ActivationComputeDataType"].toChar() + if self["ActivationNoGuard"]: name += "NG" + + if self["UseScaleAB"] == "Scalar": name += "_SAB" + elif self["UseScaleAB"] == "Vector": name += "_SABV" + if self["UseScaleCD"]: name += "_SCD" + if self["UseScaleAlphaVec"]: name += "_SAV" + + if self["SupportUserArgs"]: name += "_UserArgs" + + return name + + def keys(self): + return list(self.state.keys()) + def __len__(self): + return len(self.state) + def __iter__(self): + return iter(self.state) + def __getitem__(self, key): + return self.state[key] + def __setitem__(self, key, value): + self.state[key] = value + def __repr__(self): + return self.__str__() + def getAttributes(self): + return self.state + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): + return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + + def get(self, key, default=None): + try: + return self.state[key] + except: + return default + +################################################################################ +# Bias Type +################################################################################ + +def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: + bList = [] + for d in ["DataType", "ComputeDataType", "DestDataType"]: + dtype = DataType(problem[d]) + # filter out int8, because it is not supported by bias datatype + # TODO + if not dtype.isInt8(): + bList.append(dtype) + + biasDataTypeList = list(set(bList)) + biasDataTypeList.sort() # Make name unique + return biasDataTypeList \ No newline at end of file diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index c7bdb8d378..8e8d3bc402 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -36,7 +36,7 @@ from .AsmStoreState import VectorDataTypes from .Common import assignParameterWithDefault, IsaInfo, \ - defaultProblemType, defaultSolution, \ + defaultSolution, \ defaultInternalSupportParams, \ internalParameters, \ print2, printExit, printWarning, \ @@ -44,6 +44,7 @@ roundUp, validWMMA, \ INDEX_CHARS, IsaVersion, SemanticVersion, \ DepthUConfig, getNameFull +from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler from collections import OrderedDict @@ -57,76 +58,6 @@ import operator import sys - -# The supported typed GEMM, each entry is (Ti, To, Tc). -# DataType (Ti) = The data-type of the input matrices: A/B -# DestDataType (To) = The data-type of the output matrices: C/D -# ComputeDataType (Tc) = The data-type of computation: alpha/beta: -# Cinternal: basically should == ComputeDataType - -# This is used in SolutionStruct.py::checkIfSupportedGEMMType() -validGEMMTypes = [ - ("H", "H", "H"), - ("S", "S", "S"), - ("D", "D", "D"), - ("C", "C", "C"), - ("Z", "Z", "Z"), - ("H", "H", "S"), - ("H", "S", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I8", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("B8", "B", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), - ("B8F8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), # F8 NANOO - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("B8N", "B", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), - ("B8F8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - # All HPA types are listed here (HPA=T). The name of the library logic files for these types is: # *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. # The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. @@ -219,505 +150,6 @@ class Fbs(Enum): Batch=1 # Expect to be batch dimension Sum=2 # Expect to be summation dimension -################################################################################ -# ProblemType -# name of solution should begin with name of problemType, and arguments can be listed out explicitly -class ProblemType(Mapping): - ######################################## - def __init__(self, config, printIndexAssignmentInfo: bool): - self.state = {} - - for key in defaultProblemType: - assignParameterWithDefault(self.state, key, config, defaultProblemType) - - # adjusting all data types - if "DataType" in config: - self["DataType"] = DataType(config["DataType"]) - self["DataTypeA"] = self["DataType"] - self["DataTypeB"] = self["DataType"] - else: - printExit("NO data type specified") - self["DataType"] = DataType(0) - self["DataTypeA"] = DataType(0) - self["DataTypeB"] = DataType(0) - - if "DataTypeA" in config: - self["DataTypeA"] = DataType(config["DataTypeA"]) - - if "DataTypeB" in config: - self["DataTypeB"] = DataType(config["DataTypeB"]) - - if "DestDataType" in config: - self["DestDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["DestDataType"] = DataType(config["DataType"]) - else: - printExit("NO dest data type or data type specified") - self["DataType"] = DataType(0) - - self["DataTypeE"] = self["DestDataType"] - if "DataTypeE" in config: - self["DataTypeE"] = DataType(config["DataTypeE"]) - - if "ComputeDataType" in config: - self["ComputeDataType"] = DataType(config["ComputeDataType"]) - else: - if "DestDataType" in config: - self["ComputeDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["ComputeDataType"] = DataType(config["DataType"]) - else: - printExit("NO compute data type, or dest data type, or data type specified") - self["DataType"] = DataType(0) - - # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. - # So far we don't have to set it in config yamls - self["DataTypeAmaxD"] = self["ComputeDataType"] - if "DataTypeAmaxD" in config: - self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) - - if self["Sparse"]: - self["DataTypeMetadata"] = DataType("I8") - - if "F32XdlMathOp" in config: - self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) - else: - self["F32XdlMathOp"] = DataType(0) - - # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. - if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: - print2("DataType == i8 and HPA == True; setting compute data type to int32") - self["ComputeDataType"] = DataType('i') - - if self["OperationType"] == "GEMM": - self.checkIfSupportedGEMMType() - self.initGEMM() - else: - printExit("Unsupported OperationType = %s" % self["OperationType"]) - - self.state["AssignedDerivedParameters"] = False - ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) - - for tc in ('A', 'B'): - for sc in self["SetConstStride%s"%tc] : - (anchorDim, stride) = sc[:2] - if anchorDim not in self.state["IndexAssignments%s"%tc]: - printExit("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) - - # Bias - # If compute data type is not equal to dest data type, tensile will run conversion kernel. - # In this case we don't need to apply bias in beta only kernel. - if "UseBias" in config: - if self["ComputeDataType"] != self["DestDataType"]: - self["BetaOnlyUseBias"] = False - else: - self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False - if "BiasDataTypeList" in config: - self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] - self["BiasDataTypeList"].sort() # Make name unique - else: - self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) - else: - self["BetaOnlyUseBias"] = False - self["BiasDataTypeList"] = [] - - # Activation - # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. - # Otherwise, ActivationType will be set to 'none'. - if "Activation" in config: - typeStr = config.get("ActivationType", 'none') - if typeStr not in ['all', 'hipblaslt_all']: - typeStr = 'none' - else: - typeStr = 'none' - self["ActivationType"] = ActivationType(typeStr) - if "ActivationComputeDataType" in config: - self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) - else: - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if self["ActivationType"] != 'none': - # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type - if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: - printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ - ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ - self["ComputeDataType"], \ - self["DestDataType"])) - self["ActivationComputeDataType"] = self["ComputeDataType"] - if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ - (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): - printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ - ActivationComputeDataType will be set to ComputeDataType automatically.") - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if "UseE" in config: - if config["UseE"]: - if self["ActivationType"] == 'none': - printWarning("Use E is disabled cause Activation is set to False.") - self["UseE"] = False - else: - self["UseE"] = config["UseE"] - else: - self["UseE"] = config["UseE"] - - if "Gradient" in config: - if config["Gradient"]: - if (not self["UseBias"]) and self["ActivationType"] == 'none': - printWarning("Gradient is disabled cause bias and activation are both disabled.") - self["Gradient"] = False - if self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is enabled cause Activation is enabled.") - self["UseE"] = True - elif self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is disabled cause Activation is disabled.") - self["UseE"] = False - # if self["UseScaleAlphaVec"]: - # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") - # self["UseScaleAlphaVec"] = False - self["Gradient"] = config["Gradient"] - - # Need gradient info - biasSrcList = ["A", "B", "D"] - if "BiasSrc" in config: - if not self["Gradient"] and config["BiasSrc"] != "D": - printWarning("BiasSrc is set to D cause Gradient is disabled.") - self["BiasSrc"] = "D" - elif self["Gradient"]: - # # Currently only supports D :) - # if config["BiasSrc"] != "D": - # printExit("BiasSrc currently only supports D.") - if config["BiasSrc"] not in biasSrcList: - printExit("BiasSrc only supports A, B, D.") - - if "ActivationNoGuard" in config: - self["ActivationNoGuard"] = config["ActivationNoGuard"] - if self["ActivationNoGuard"]: - if self["ActivationType"] == 'none': - printWarning("ActivationNoGuard is set to False cause Acivation is off.") - self["ActivationNoGuard"] = False - if (not self["Gradient"]): - printWarning("ActivationNoGuard is set to False cause Gradient is off.") - self["ActivationNoGuard"] = False - - ################################################################################ - # Function checkIfSupportedGEMMType: - # Assures 3 data-types are valid, supported and well-assigned - # See the discussion on Common.py for validGEMMTypes - ################################################################################ - def checkIfSupportedGEMMType(self): - inType = self["DataType"] - outType = self["DestDataType"] - computeType = self["ComputeDataType"] - - gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) - if gemmType not in validGEMMTypes: - printExit("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0],gemmType[1],gemmType[2])) - - ######################################## - def initGEMM(self): - sumIdx = 3 if self["Batched"] else 2 - self["IndexAssignmentsA"] = [0, sumIdx] # N - self["IndexAssignmentsB"] = [sumIdx, 1] # N - if self.state["Sparse"] == 2: - self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) - else: - self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) - if self["TransposeA"]: - self["IndexAssignmentsA"] = [sumIdx, 0] # T - if self["TransposeB"]: - self["IndexAssignmentsB"] = [1, sumIdx] # T - if self["Batched"]: - self["IndexAssignmentsA"].append(2) - self["IndexAssignmentsB"].append(2) - self["IndexAssignmentsMetadata"].append(2) - self["NumIndicesC"] = 3 - else: - self["NumIndicesC"] = 2 - - self["NumIndicesLD"] = 4 - self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 - for i in range(1, len(self["IndexAssignmentsLD"])): - self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 - - ######################################## - def isGEMM(self): - return self.operationType == 0 - - ######################################## - # determine d0, d1, dU - @staticmethod - def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): - if "AssignedDerivedParameters" in state: - if state["AssignedDerivedParameters"]: - return - state["AssignedDerivedParameters"] = False - - state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ - max(state["IndexAssignmentsB"])+1) - - # determine num free, batch - state["IndicesFree"] = [] - state["IndicesBatch"] = [] - state["IndicesSummation"] = [] - - for i in range(0, state["NumIndicesC"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesBatch"].append(i) - - elif inA or inB: - state["IndicesFree"].append(i) - else: - printExit("invalid index %u (inC but not (inA or inB))" % i) - - # determine num summation - for i in range(state["NumIndicesC"], state["TotalIndices"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesSummation"].append(i) - else: - printExit("invalid index %u (expected summation but not (inA and inB))" % i) - # print index assignments - if printIndexAssignmentInfo: - print("IndicesFree: %s" % state["IndicesFree"]) - print("IndicesBatch: %s" % state["IndicesBatch"]) - print("IndicesSum: %s" % state["IndicesSummation"]) - print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) - print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) - print("NumIndicesC: %s" % state["NumIndicesC"]) - - for k in ('IndexAssignmentsA','IndexAssignmentsB'): - if len(state[k]) != len(set(state[k])): - printExit("duplicate index in %s=%s"% (k,state[k])) - - state["NumIndicesFree"] = len(state["IndicesFree"]) - state["NumIndicesBatch"] = len(state["IndicesBatch"]) - state["NumIndicesSummation"] = len(state["IndicesSummation"]) - if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : - printExit("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) - - # by default, unroll index will be the last/inner summation index - state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] - for i in range(0, len(state["IndexAssignmentsA"])): - if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: - state["IndexUnrollA"] = i - break - for i in range(0, len(state["IndexAssignmentsB"])): - if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: - state["IndexUnrollB"] = i - break - for i in range(0, len(state["IndexAssignmentsMetadata"])): - if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: - state["IndexUnrollM"] = i - break - #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) - #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) - - # assign d0, d1 - if state["AllowNoFreeDims"]: - dimList = state["IndicesFree"] + state["IndicesBatch"] - else: - dimList = state["IndicesFree"] - state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] - state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] - #print2("Index01A: %u" % state["Index01A"]) - #print2("Index01B: %u" % state["Index01B"]) - # Store code is optimized for 0 as the fastest-moving in memory - # whichever has lower stride in C (lower value), is 0, other is 1 - if state["Index01A"] < state["Index01B"]: - state["Index0"] = state["Index01A"] - state["Index1"] = state["Index01B"] - state["Tensor0"] = 0 - state["Tensor1"] = 1 - state["TileA"] = 0 - state["TileB"] = 1 - else: - state["Index0"] = state["Index01B"] - state["Index1"] = state["Index01A"] - state["Tensor0"] = 1 - state["Tensor1"] = 0 - state["TileA"] = 1 - state["TileB"] = 0 - - # generalize transpose - strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) - strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) - unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) - unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) - state["TLUA"] = strideIdxA < unrollIdxA - state["TLUB"] = strideIdxB < unrollIdxB - #state["TLUB"] = True # hack - - if printIndexAssignmentInfo: - print("TLUA: %s (stridePosA(%d) 1 : - name += "_FD%s"%("N" if factorDim == 2 else "MN") - - if self["UseE"]: - if self["Gradient"]: - name += "_Grad%s"%self["DataTypeE"].toChar() - else: - name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types - if self["OutputAmaxD"]: - name += "_AmaxD" - if self["Sparse"]: - if self["Sparse"] == 2: - name += "_SPB" - else: - name += "_SPA" - - # precision and other - # name += "_SB" if self["StridedBatched"] else "_GB" - if self["GroupedGemm"]: - name += "_GG" - else: - name += "" if self["StridedBatched"] else "_GB" # legacy - - # Activation Naming - if self["ActivationType"] != 'none': - if self["ActivationType"] == 'all': - name += "_A" - elif self["ActivationType"] == 'hipblaslt_all': - name += "_HA" - else: - name += "_%s"%str(self["ActivationType"]).upper() - name += self["ActivationComputeDataType"].toChar() - if self["ActivationNoGuard"]: name += "NG" - - if self["UseScaleAB"] == "Scalar": name += "_SAB" - elif self["UseScaleAB"] == "Vector": name += "_SABV" - if self["UseScaleCD"]: name += "_SCD" - if self["UseScaleAlphaVec"]: name += "_SAV" - - if self["SupportUserArgs"]: name += "_UserArgs" - - return name - - def keys(self): - return list(self.state.keys()) - def __len__(self): - return len(self.state) - def __iter__(self): - return iter(self.state) - def __getitem__(self, key): - return self.state[key] - def __setitem__(self, key, value): - self.state[key] = value - def __repr__(self): - return self.__str__() - def getAttributes(self): - return self.state - def __hash__(self): - return hash(str(self)) - def __eq__(self, other): - return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() - def __ne__(self, other): - result = self.__eq__(other) - if result is NotImplemented: - return result - return not result - - def get(self, key, default=None): - try: - return self.state[key] - except: - return default - - ################################################################################ # ProblemSizeRange @@ -1083,23 +515,6 @@ def __str__(self): s = "FactorDimArgs\n" return s -################################################################################ -# Bias Type -################################################################################ - -def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: - bList = [] - for d in ["DataType", "ComputeDataType", "DestDataType"]: - dtype = DataType(problem[d]) - # filter out int8, because it is not supported by bias datatype - # TODO - if not dtype.isInt8(): - bList.append(dtype) - - biasDataTypeList = list(set(bList)) - biasDataTypeList.sort() # Make name unique - return biasDataTypeList - class BiasTypeArgs: ######################################## @@ -1204,7 +619,7 @@ def __init__( if "ProblemType" in config: self["ProblemType"] = ProblemType(config["ProblemType"], printIndexAssignmentInfo) else: - self["ProblemType"] = ProblemType(defaultProblemType, printIndexAssignmentInfo) + self["ProblemType"] = ProblemType.FromDefaultConfig(printIndexAssignmentInfo) if "InternalSupportParams" in config: self["InternalSupportParams"] = {} diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py index da6d2f1377..51cfdf41cf 100644 --- a/tensilelite/Tensile/TensileClientConfig.py +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -25,7 +25,8 @@ from . import ClientWriter from . import LibraryIO from .Contractions import ProblemType as ContractionsProblemType -from .SolutionStructs import ProblemSizes, ProblemType +from .SolutionStructs import ProblemSizes +from Tensile.ProblemType import ProblemType from .Common import globalParameters, print1, printExit, printWarning, assignGlobalParameters, \ restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 86d8edc580..d3caad4636 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -29,36 +29,33 @@ import shutil from pathlib import Path from timeit import default_timer as timer -from typing import List, NamedTuple, Optional, Sequence, Union +from typing import List, NamedTuple, Optional, Union from Tensile import SOURCE_PATH, LibraryIO from Tensile.Common import ( + assignGlobalParameters, CHeader, DebugConfig, DepthUConfig, + ensurePath, + globalParameters, getKernelFileBase, getKeyNoInternalArgs, getMinNaming, getSerialNaming, gfxToIsa, HR, - IsaInfo, + isaToGfx, IsaVersion, makeIsaInfoMap, ParallelMap2, - SemanticVersion, - architectureMap, - globalParameters, - assignGlobalParameters, - ensurePath, - isaToGfx, print1, print2, printWarning, printExit, printWarning, state, - SUPPORTED_ISA, + SUPPORTED_GFX, tqdm, verbosity, ) @@ -566,6 +563,8 @@ def run(): archs = arguments["Architecture"].split(";") else: archs = arguments["Architecture"].split("_") + archs = archs if archs == "all" else SUPPORTED_GFX + targetIsas = [gfxToIsa(a) for a in archs] isaInfoMap = makeIsaInfoMap(targetIsas, cxxCompiler) assignGlobalParameters(arguments, isaInfoMap, cxxCompiler) @@ -590,13 +589,6 @@ def run(): if not os.path.exists(arguments["LogicPath"]): printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") - logicArchs = set() - for arch in archs: - if arch in architectureMap: - logicArchs.add(architectureMap[arch]) - else: - printExit("Architecture %s not supported" % arch) - logicExtFormat = ".yaml" if arguments["LogicFormat"] == "yaml": pass From 641eeb9c1d03b0008373f4444cf04aeeff6494f3 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 18 Feb 2025 16:02:18 +0000 Subject: [PATCH 072/105] Break dependency on printExit --- tensilelite/Tensile/Common/ValidParameters.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index 5d0a61d636..fad16def34 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -25,7 +25,6 @@ import math from .Architectures import SUPPORTED_ISA -from .Utilities import printExit ################################################################################ # Enumerate Valid Solution Parameters @@ -795,7 +794,7 @@ def checkParametersAreValid(param, validParams): return if name not in validParams: - printExit( + raise Exception( "Invalid parameter name: {}\nValid parameters are {}.".format( name, sorted(validParameters.keys()) ) @@ -809,4 +808,4 @@ def checkParametersAreValid(param, validParams): if len(validParams[name]) > 32 else "" ) - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) + raise Exception(msgBase.format(name, value, name, validParams[name][:32], msgExt)) From c1cc39ec76eac841942206300a321561e6ed6338 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 18 Feb 2025 16:05:48 +0000 Subject: [PATCH 073/105] Ordering --- tensilelite/Tensile/Common/Architectures.py | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index 9f885461cb..7559142554 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -76,8 +76,24 @@ IsaVersion(12, 0, 1), ] + +def isaToGfx(arch: IsaVersion) -> str: + """Converts an ISA version to a gfx architecture name. + + Args: + arch: An object representing the major, minor, and step version of the ISA. + + Returns: + The name of the GPU architecture (e.g., 'gfx906'). + """ + # Convert last digit to hex because reasons + name = str(arch[0]) + str(arch[1]) + ("%x" % arch[2]) + return "gfx" + "".join(map(str, name)) + + SUPPORTED_GFX = [isaToGfx(isa) for isa in SUPPORTED_ISA] + def gfxToIsa(name: str) -> Optional[IsaVersion]: """Extracts the ISA version from a given gfx architecture name. @@ -102,20 +118,6 @@ def gfxToIsa(name: str) -> Optional[IsaVersion]: return IsaVersion(major, minor, step) -def isaToGfx(arch: IsaVersion) -> str: - """Converts an ISA version to a gfx architecture name. - - Args: - arch: An object representing the major, minor, and step version of the ISA. - - Returns: - The name of the GPU architecture (e.g., 'gfx906'). - """ - # Convert last digit to hex because reasons - name = str(arch[0]) + str(arch[1]) + ("%x" % arch[2]) - return "gfx" + "".join(map(str, name)) - - def gfxToSwCodename(gfxName: str) -> Optional[str]: """Retrieves the common name for a given gfx architecture name. From 061ffb6674a78a3d93586c507514d8511e329b81 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 18 Feb 2025 16:20:57 +0000 Subject: [PATCH 074/105] Fix missing state --- tensilelite/Tensile/ProblemType.py | 61 +++++++++++++++++++++++++- tensilelite/Tensile/SolutionStructs.py | 55 ----------------------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py index 2787d0195f..27c3e9e9ea 100644 --- a/tensilelite/Tensile/ProblemType.py +++ b/tensilelite/Tensile/ProblemType.py @@ -27,8 +27,11 @@ from Tensile.Activation import ActivationType from Tensile.TensileInstructions.DataType import DataType + +from Tensile.Common.Constants import INDEX_CHARS from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2 + ################################################################################ # ProblemType # name of solution should begin with name of problemType, and arguments can be listed out explicitly @@ -194,6 +197,62 @@ ] +# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: +# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. +# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. +_HPATypes = [ + ("H", "S", "S"), + ("H", "H", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + class ProblemType(Mapping): ######################################## @@ -591,7 +650,7 @@ def __str__(self): # HHS, HSS, BSS and I8II kernels, use a clearer naming _TiToTc_ # TODO: Distinguish all kernels by _TiToTc_ to be more consistent with rocblas gemmType = (self["DataType"].toChar(),self["DestDataType"].toChar(),self["ComputeDataType"].toChar() ) - if gemmType in HPATypes: + if gemmType in _HPATypes: name += self["DestDataType"].toChar() # Type of C/D name += self["ComputeDataType"].toChar() # Type of Alpha/Beta name += "_" diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 8e8d3bc402..d5a8251966 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -58,61 +58,6 @@ import operator import sys -# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: -# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. -# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. -HPATypes = [ - ("H", "S", "S"), - ("H", "H", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - ######################################## # Print a reject message : From 6da643ac3926c0b2b20048af5348934ca23477d2 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 18 Feb 2025 19:40:11 +0000 Subject: [PATCH 075/105] fix logic --- tensilelite/Tensile/TensileCreateLibrary/Run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index d3caad4636..393474bffd 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -563,7 +563,7 @@ def run(): archs = arguments["Architecture"].split(";") else: archs = arguments["Architecture"].split("_") - archs = archs if archs == "all" else SUPPORTED_GFX + archs = SUPPORTED_GFX if archs == "all" else archs targetIsas = [gfxToIsa(a) for a in archs] isaInfoMap = makeIsaInfoMap(targetIsas, cxxCompiler) From 1b2967c3bddc23d7f4fa2067d1704fdc1fb87e31 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Wed, 19 Feb 2025 18:19:45 +0000 Subject: [PATCH 076/105] Fix build errors --- tensilelite/Tensile/BenchmarkProblems.py | 6 +++--- tensilelite/Tensile/Common/__init__.py | 2 +- tensilelite/Tensile/KernelWriter.py | 3 ++- tensilelite/Tensile/KernelWriterAssembly.py | 6 +++--- tensilelite/Tensile/LibraryLogic.py | 4 ++-- tensilelite/Tensile/SolutionLibrary.py | 3 ++- tensilelite/Tensile/SolutionSelectionLibrary.py | 2 +- tensilelite/Tensile/SolutionStructs.py | 4 ++-- tensilelite/Tensile/TensileCreateLibrary/Run.py | 6 ++---- tensilelite/Tensile/TensileInstructions/Base.py | 3 ++- 10 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 0bba1b949d..4467758caf 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -47,9 +47,9 @@ from Tensile.Toolchain.Component import Assembler from .Common import globalParameters, HR, print1, print2, IsaInfo, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ - BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, \ - DepthUConfig, getMinNaming, getNameMin, getSerialNaming, getNameFull, \ - getKeyNoInternalArgs + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, DepthUConfig +from Tensile.Common.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ + getKeyNoInternalArgs def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index b0a4e616f0..e2dd14fe6a 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -5,7 +5,7 @@ # Dunder variables are not exported via `*` from .GlobalParameters import * from .GlobalParameters import __version__ -from .Naming import * +#from .Naming import * from .Parallel import * from .Types import * from .Utilities import * diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 15c66e4e70..702010a27a 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,7 +36,8 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ - INDEX_CHARS, IsaVersion, getKernelName + INDEX_CHARS, IsaVersion +from Tensile.Common.Naming import getKernelName from Tensile.Toolchain.Component import Assembler import abc diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 07475dcb00..0391f1e6cd 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,8 +48,8 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection, \ - SemanticVersion, IsaVersion, IsaInfo, getKernelFileBase +from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection +from Tensile.Common.Naming import getKernelFileBase from Tensile.Toolchain.Component import Assembler from math import ceil, log, floor @@ -94,7 +94,7 @@ def __init__( def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): - kernelName = getKernelFileBase(useShortNames, self.kernelSerialNaming, kernel) + kernelName = getKernelFileBase(useShortNames, self.debugConfig.splitGSU, self.kernelMinNaming, self.kernelSerialNaming, kernel) with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: rocmVersion = self.assembler.rocm_version if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650): diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 6eb2899089..8b3b4f0e90 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -26,8 +26,8 @@ from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig, \ - getMinNaming, getNameMin, getNameFull + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig +from Tensile.Common.Naming import getMinNaming, getNameMin, getNameFull from . import LibraryIO from . import SolutionSelectionLibrary diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index aca9ffb01d..96fcbe2ab2 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -29,7 +29,8 @@ from . import Hardware from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state, IsaInfo, gfxToIsa, DepthUConfig, getMinNaming, getNameMin +from .Common import state, IsaInfo, gfxToIsa, DepthUConfig +from Tensile.Common.Naming import getMinNaming, getNameMin class SingleSolutionLibrary: Tag = "Single" diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py index d3944d309c..49a8115e76 100644 --- a/tensilelite/Tensile/SolutionSelectionLibrary.py +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -22,7 +22,7 @@ # ################################################################################ -from Tensile.Common import getNameMin +from Tensile.Common.Naming import getNameMin import csv diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index d5a8251966..0f70c2f0b6 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -43,14 +43,14 @@ validMFMA, validSMFMA, \ roundUp, validWMMA, \ INDEX_CHARS, IsaVersion, SemanticVersion, \ - DepthUConfig, getNameFull + DepthUConfig +from Tensile.Common.Naming import getNameFull from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler from collections import OrderedDict from collections.abc import Mapping from enum import Enum -from functools import lru_cache from typing import List, Dict import collections diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 393474bffd..a31783dc9a 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -39,10 +39,6 @@ DepthUConfig, ensurePath, globalParameters, - getKernelFileBase, - getKeyNoInternalArgs, - getMinNaming, - getSerialNaming, gfxToIsa, HR, isaToGfx, @@ -59,6 +55,8 @@ tqdm, verbosity, ) +from Tensile.Common.Naming import getKernelFileBase, getKeyNoInternalArgs, getMinNaming, getSerialNaming + from Tensile.CustomYamlLoader import load_logic_gfx_arch from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import ( diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index 27fbb0f812..86397e12ea 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -27,7 +27,8 @@ from dataclasses import dataclass from typing import Tuple -from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs, IsaInfo, IsaVersion +from Tensile.Common.Capabilities import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from Tensile.Common.Types import IsaInfo, IsaVersion from .Formatting import __TI_DEBUG_LEVEL__, printExit from timeit import default_timer as timer From 9a2c213825f75ffa59893f97bac766c9fd552862 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Fri, 21 Feb 2025 19:50:33 +0000 Subject: [PATCH 077/105] Fix if test --- tensilelite/Tensile/KernelWriter.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index d45320983e..84566ccd05 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -2326,11 +2326,14 @@ def _loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loo ## 8x8 -> split into group of 16 MAC(s) ## supports only PLR=0 ############################################################################### - if self.states.numItersPLR: - subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ - u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) - module.add(subIterCode) # add scheduled "other", local reads, local writes - pack[luIdx] = Module() + + # Is this test necessary because of the global variable this if was previously always true + # after removing the global variable it is always false... + # if self.states.numItersPLR: + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) + module.add(subIterCode) # add scheduled "other", local reads, local writes + pack[luIdx] = Module() # close unrolled loop endStr = "" From 7a978722d149c47159e08cc3f18367db341514d1 Mon Sep 17 00:00:00 2001 From: David Dixon Date: Tue, 25 Feb 2025 18:40:46 +0000 Subject: [PATCH 078/105] Need to use all kernels when building co files --- tensilelite/Tensile/TensileCreateLibrary/Run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index a31783dc9a..eba0c1bd76 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -366,7 +366,7 @@ def assemble(ret): asmToolchain.linker, asmToolchain.bundler, globalParameters["ROCmLdPath"], - uniqueAsmKernels, + asmKernels, destLibPath, assemblyTmpPath, compress, From a2541dc121c577522637f13bc1c2eb708fbe2e90 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Tue, 18 Feb 2025 22:09:10 -0600 Subject: [PATCH 079/105] refactor: valid parameter checks --- tensilelite/Tensile/BenchmarkProblems.py | 32 ++- tensilelite/Tensile/BenchmarkStructs.py | 25 +- tensilelite/Tensile/CustomKernels.py | 38 ++- tensilelite/Tensile/SolutionStructs.py | 157 +++++------ .../Tensile/TensileLogic/ParseArguments.py | 7 +- tensilelite/Tensile/TensileLogic/Run.py | 139 +++++++--- tensilelite/Tensile/TensileLogic/Utilities.py | 34 +++ .../TensileLogic/ValidMatrixInstruction.py | 243 ++++++++++++------ .../Tensile/TensileLogic/ValidWorkGroup.py | 58 +++++ 9 files changed, 498 insertions(+), 235 deletions(-) create mode 100644 tensilelite/Tensile/TensileLogic/Utilities.py create mode 100644 tensilelite/Tensile/TensileLogic/ValidWorkGroup.py diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 4467758caf..a055b045a1 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -33,6 +33,8 @@ from typing import Dict from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO +from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters + from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni @@ -64,17 +66,25 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse solution.update(constantParams) solution.update(perm) - # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution( - solution, - debugConfig.splitGSU, - debugConfig.printSolutionRejectionReason, - debugConfig.printIndexAssignmentInfo, - depthUConfig, - assembler, - isaInfoMap - ) - if solutionObject["Valid"]: + mi = solution["MatrixInstruction"] + isa = solution["ISA"] + wavefrontSize = solution["WavefrontSize"] + enableF32x = solution.get("EnableF32XdlMathOp", False) + + miParams = Solution.matrixInstructionToMIParameters(mi, isa, wavefrontSize, problemType, enableF32x) + solution.update(miParams) + validateMIParameters(solution, globalParameters) + + if solution["Valid"]: + solutionObject = Solution( + solution, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap + ) if solutionObject not in solutionSet: solutionSet.add(solutionObject) solutions.append(solutionObject) diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index 0d2b7cf76c..55637fb787 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,8 @@ import itertools from .Common import print1, print2, hasParam, printExit, \ defaultBenchmarkCommonParameters, validParameters, globalParameters, \ - defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes + defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes, \ + checkParametersAreValid from .CustomKernels import getAllCustomKernelNames from .SolutionStructs import ProblemSizes, ActivationArgs, BiasTypeArgs, \ FactorDimArgs @@ -44,26 +45,6 @@ def getDefaultsForMissingParameters(paramList, defaultParams): return benchmarkParams -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit("Invalid parameter name: {}\nValid parameters are {}." \ - .format(name, sorted(validParameters.keys()))) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ - if len(validParams[name])>32 else "" - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) - - def separateParameters(paramSetList): """Separates paramSetList into parameters with single and multiple values""" singleVaules = {} diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index 12a8214eb9..e31a562d3e 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -65,20 +65,38 @@ def readCustomKernelConfig(name, directory=CUSTOM_KERNEL_PATH): except yaml.scanner.ScannerError as e: raise RuntimeError("Failed to read configuration for custom kernel: {0}\nDetails:\n{1}".format(name, e)) -def getCustomKernelConfig(kernelName, internalSupportParams, directory=CUSTOM_KERNEL_PATH): +def getCustomKernelConfig( + kernelName: str, internalSupportParams: dict, directory: str = CUSTOM_KERNEL_PATH +) -> dict: + """ + Retrieves and validates the configuration for a custom kernel. + + Args: + kernelName: The name of the custom kernel. + internalSupportParams: A dictionary of internal support parameters to be merged with the kernel configuration. + directory: The directory where custom kernel files are located. Defaults to CUSTOM_KERNEL_PATH. + + Returns: + dict: The validated configuration dictionary for the custom kernel. + + Raises: + RuntimeError: If the custom kernel configuration is missing required fields or if there is an error reading the configuration. + """ kernelConfig = readCustomKernelConfig(kernelName, directory) if "InternalSupportParams" not in kernelConfig: - raise RuntimeError("Custom kernel %s config must have KernArgsVersion"%kernelName) - else: - # CustomKernelConfig must have signature version - if "KernArgsVersion" not in kernelConfig["InternalSupportParams"]: - raise RuntimeError("Custom kernel %s config must have KernArgsVersion"%kernelName) - for key in internalSupportParams: - if key not in kernelConfig["InternalSupportParams"]: - kernelConfig["InternalSupportParams"][key] = internalSupportParams[key] + raise RuntimeError(f"Custom kernel {kernelName} config must have 'InternalSupportParams'") + + if "KernArgsVersion" not in kernelConfig["InternalSupportParams"]: + raise RuntimeError(f"Custom kernel {kernelName} config must have 'KernArgsVersion'") + + kernelIsp = kernelConfig["InternalSupportParams"] + for key in internalSupportParams: + if key not in kernelIsp: + kernelIsp[key] = internalSupportParams[key] for k, v in kernelConfig.items(): if k != "ProblemType": checkParametersAreValid((k, [v]), validParameters) + kernelConfig["KernelLanguage"] = "Assembly" kernelConfig["CustomKernelName"] = kernelName diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 810f1098cb..a693f0eb7c 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -35,6 +35,7 @@ from .Activation import ActivationType from .AsmStoreState import VectorDataTypes +from .CustomKernels import isCustomKernelConfig from .Common import assignParameterWithDefault, IsaInfo, \ defaultSolution, \ defaultInternalSupportParams, \ @@ -48,6 +49,7 @@ from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler + from collections import OrderedDict from collections.abc import Mapping from enum import Enum @@ -55,9 +57,10 @@ import collections import math -import operator import sys +count = 0 + ######################################## # Print a reject message : @@ -819,7 +822,6 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): reject(state, printRejectionReason, "General Batched GEMM only support GEMM OperationType") - Solution.MatrixInstructionToMIParameters(state, printRejectionReason, isaInfoMap) EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None if EnableMatrixInstruction == None: if ("MIBlock" in state and len(state["MIBlock"]) == 6) \ @@ -1024,78 +1026,83 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota @staticmethod - def MatrixInstructionToMIParameters(state, printRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): - isa = state["ISA"] - if len(state["MatrixInstruction"]) == 9: - mi = state["MatrixInstruction"] - state["MatrixInstruction"] = [state["MatrixInstruction"][0],state["MatrixInstruction"][1],state["MatrixInstruction"][2],state["MatrixInstruction"][3]] - - waves = mi[7]* mi[8] - miwg0 = mi[4] * mi[0] * mi[7] - state["WorkGroup"][0] = miwg0 - state["WorkGroup"][1] = waves*state["WavefrontSize"] // state["WorkGroup"][0] - state["ThreadTile"][0] = 1 # dummy - state["ThreadTile"][1] = 1 # dummy - - state["MFMA_BF16_1K"] = False - if not state["ProblemType"]["Sparse"]: - miDataType = state["ProblemType"]["DataType"] if (not state["EnableF32XdlMathOp"]) else state["ProblemType"]["F32XdlMathOp"] - if isaInfoMap[isa].asmCaps["HasMFMA"]: - if not (miDataType.toChar() in validMFMA and \ - state["MatrixInstruction"] in validMFMA[miDataType.toChar()]): - if miDataType.isBFloat16() and \ - state["MatrixInstruction"] in validMFMA["B1k"]: - state["MFMA_BF16_1K"] = True - else: - reject(state, printRejectionReason, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], miDataType)) - elif isaInfoMap[isa].asmCaps["HasWMMA"]: - if state["MatrixInstruction"] not in validWMMA: - reject(state, printRejectionReason, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) - else: - if not (state["ProblemType"]["DataType"].toChar() in validSMFMA and \ - state["MatrixInstruction"] in validSMFMA[state["ProblemType"]["DataType"].toChar()]): - reject(state, printRejectionReason, "Sparse MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) - - # set EnableMatrixInstruction - state["EnableMatrixInstruction"] = True - - # set MIBlock - MIBlock_BM = miwg0 // mi[0] - MIBlock_BM = min(MIBlock_BM, mi[3]) - MIBlock_BN = mi[3] // MIBlock_BM - - state["MIBlock"] = [32, 32, 2, 1, 1, 1] - state["MIBlock"][0] = mi[0] - state["MIBlock"][1] = mi[1] - state["MIBlock"][2] = mi[2] - state["MIBlock"][3] = mi[3] - state["MIBlock"][4] = MIBlock_BM - state["MIBlock"][5] = MIBlock_BN - - # set MIWaveGroup - state['MIWaveGroup'] = [1, 1] - state['MIWaveGroup'][0] = min((miwg0 // mi[0]) // MIBlock_BM, waves) - state['MIWaveGroup'][1] = waves // state['MIWaveGroup'][0] - - # set MIWaveTile - state['MIWaveTile'] = [1, 1] - state['MIWaveTile'][0] = mi[5] - state['MIWaveTile'][1] = mi[6] - # set MIInputPerThread - isa = tuple(state["ISA"]) - state['MIInputPerThread'] = state["MatrixInstruction"][0] * state["MatrixInstruction"][2] * state["MatrixInstruction"][3] // state["WavefrontSize"] - if (not isaInfoMap[isa].asmCaps['HasMFMA']) and isaInfoMap[isa].asmCaps['HasWMMA']: - if state['ISA'][0] == 10 or state['ISA'][0] == 11: - state['MIInputPerThread'] = state["MatrixInstruction"][2] - sparseA = False if not state["ProblemType"]["Sparse"] else False if state["ProblemType"]["Sparse"] == 2 else True - sparseB = False if not state["ProblemType"]["Sparse"] else True if state["ProblemType"]["Sparse"] == 2 else False - state['MIInputPerThreadA'] = state['MIInputPerThread'] if not sparseA else state['MIInputPerThread']//2 - state['MIInputPerThreadB'] = state['MIInputPerThread'] if not sparseB else state['MIInputPerThread']//2 - state['MIInputPerThreadMetadata'] = state['MIInputPerThread'] if not state["ProblemType"]["Sparse"] else state['MIInputPerThread']//8 - elif state["MatrixInstruction"] != [] and len(state["MatrixInstruction"]) == 4: - state["EnableMatrixInstruction"] = True - else: - state["EnableMatrixInstruction"] = False + def matrixInstructionToMIParameters( + mi: list, + isa: IsaVersion, + wavefrontSize: int, + problemType: dict, + enableF32x: bool, + isaInfoMap: Dict[str, IsaInfo] + ): + """ + Converts a 9-item matrix instruction into the associated 4-item alternative, along with + supporting MI parameters. + + Args: + mi: The matrix instruction to convert. Must have length 9. + isa: The ISA tuple. + wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. + problemType: The problem type dictionary. Typically "ProblemType" in a solution. + enableF32x: Whether to enable F32x. Typically "EnableF32XdlMathOp" in a solution. + """ + if len(mi) != 9: + raise ValueError(f"MatrixInstruction must be 9 items long to convert into MI" + f" Parameters, found {mi} with length {len(mi)}") + + result = {} + + mi4 = [mi[0], mi[1], mi[2], mi[3]] + result["MatrixInstruction"] = mi4 + result["EnableMatrixInstruction"] = True + + waves = mi[7]* mi[8] + miwg0 = mi[4] * mi[0] * mi[7] + + result["WorkGroup"] = [miwg0, waves*wavefrontSize // miwg0] + result["ThreadTile"] = [1, 1] + + isSparse = problemType.get(["Sparse"], 0) + miDataType = DataType( + problemType["DataType"] + if not enableF32x + else problemType["F32XdlMathOp"] + ) + + result["MFMA_BF16_1K"] = ( + not isSparse + and isaInfoMap[isa].asmCaps["HasMFMA"] + and not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]) + and miDataType.isBFloat16() + and mi4 in validMFMA["B1k"] + ) + + # set MIBlock + MIBlockBM = miwg0 // mi[0] + MIBlockBM = min(MIBlockBM, mi[3]) + MIBlockBN = mi[3] // MIBlockBM + result["MIBlock"] = [mi[0], mi[1], mi[2], mi[3], MIBlockBM, MIBlockBN] + miwg0 = min((miwg0 // mi[0]) // MIBlockBM, waves) + + # set MIWaveGroup + result['MIWaveGroup'][0] = min((miwg0 // mi[0]) // MIBlockBM, waves) + result['MIWaveGroup'][1] = waves // result['MIWaveGroup'][0] + + # set MIWaveTile + result['MIWaveTile'] = [mi[5], mi[6]] + + # set MIInputPerThread + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + result['MIInputPerThread'] = mi[0] * mi[2] * mi[3] // wavefrontSize + if (not hasMFMA) and hasWMMA and (isa[0] == 10 or isa[0] == 11): + result['MIInputPerThread'] = mi[2] + sparseA = False if not isSparse or isSparse == 2 else True + sparseB = True if isSparse == 2 else False + result['MIInputPerThreadA'] = result['MIInputPerThread'] if not sparseA else result['MIInputPerThread']//2 + result['MIInputPerThreadB'] = result['MIInputPerThread'] if not sparseB else result['MIInputPerThread']//2 + result['MIInputPerThreadMetadata'] = result['MIInputPerThread'] if not result["ProblemType"]["Sparse"] else result['MIInputPerThread']//8 + return result ############################################## @@ -3227,7 +3234,7 @@ def calcEpilogueTurns(factorDims: List) -> int: maxTurn = calcEpilogueTurns([0, 1]) vecDT.bias(0).turn = maxTurn vecDT.bias(1).turn = maxTurn - + # Calc LDS for SAV maxTurn = 0 if savDim == 1: diff --git a/tensilelite/Tensile/TensileLogic/ParseArguments.py b/tensilelite/Tensile/TensileLogic/ParseArguments.py index 25ce08cc95..2b5af8ccc9 100644 --- a/tensilelite/Tensile/TensileLogic/ParseArguments.py +++ b/tensilelite/Tensile/TensileLogic/ParseArguments.py @@ -40,12 +40,7 @@ def parseArguments() -> Dict[str, Any]: ) argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") - argParser.add_argument( - "--check-matrix-instruction", - dest="CheckMatrixInstruction", - action="store_true", - help="Checks that matrix instructions are valid for all target ISAs.", - ) + argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") argParser.add_argument( "--jobs", "-j", diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index a22eab2d22..031759085c 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -1,62 +1,143 @@ -import yaml +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + + import functools from pathlib import Path +from multiprocessing import Pool +from typing import List -from Tensile.Common import globalParameters, assignGlobalParameters, ParallelMap2 +from Tensile.Common import globalParameters, assignGlobalParameters, ParallelMap2, print1, printWarning from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain +from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig +from Tensile.SolutionStructs import Solution from .ParseArguments import parseArguments from .ValidMatrixInstruction import validateMatrixInstruction +from .ValidWorkGroup import validateWorkGroup def getParams(cxxCompiler): gp = globalParameters + assignGlobalParameters({"PrintSolutionRejectionReason": True}, cxxCompiler) + return gp - gpcache = Path.cwd() / "gpcache.yaml" - if gpcache.exists(): - with open(gpcache, "r") as f: - gp = yaml.load(f, yaml.CSafeLoader) - else: - assignGlobalParameters({}, cxxCompiler) - with open(gpcache, "w") as f: - yaml.dump(gp, f, yaml.CSafeDumper) - return gp +def handleCustomKernel(sol: dict) -> dict: + if not isCustomKernelConfig(sol): + return None + + name = sol["CustomKernelName"] + print1(f">> Custom kernel: {name}") + + custom = getCustomKernelConfig(name, {}) + sol.update(custom) + + mi = sol["MatrixInstruction"] + if len(mi) != 9: + printWarning( + f"Custom kernel {name} has {len(mi)} matrix instructions. Expected 9." + ) + + isa = sol["ISA"] + wavefrontSize = sol["WavefrontSize"] + ptype = sol["ProblemType"] + enableF32x = sol.get("EnableF32XdlMathOp", False) + + miParams = Solution.matrixInstructionToMIParameters( + mi, isa, wavefrontSize, ptype, enableF32x + ) + sol.update(miParams) + return sol -def runChecks(logicPath, gp, file): - if "Experimental" in file.parts: - return 0, 0 +def runChecks(logicPath: str, gp: dict, files: List[Path]): + """ + Run checks on the given files. + + Args: + logicPath: Path to the logic directory. + gp: Global parameters. + files: List of files to check. + + Returns: + Tuple of (keep, total) where keep is the number of solutions to keep and + total is the total number of solutions. + """ keep, total = 0, 0 - solutions = readYAML(file)[5] # Solutions are the 5th index - for s in solutions: - total += 1 - keep += validateMatrixInstruction(s, file.relative_to(logicPath), gp) - print(f">> {file.relative_to(logicPath)}") + for file in files: + if "Experimental" in file.parts: + return keep, total + + solutions = readYAML(file)[5] # Solutions are the 5th index + print1(f">> {file.relative_to(logicPath)}") + + for s in solutions: + s = handleCustomKernel(s) + if s: + keep += validateMatrixInstruction(s, gp, file.relative_to(logicPath)) + keep += validateWorkGroup(s, gp, file.relative_to(logicPath)) + else: + print1(f">> Skipping non-custom kernel...") + total += 1 return keep, total def main(): args = parseArguments() + if not any([args.Check]): + print1("No checks specified. Exiting.") + exit(0) + + jobs = int(args.Jobs) cxxCompiler = validateToolchain(args.CxxCompiler) - gp = getParams(cxxCompiler) + + gp = globalParameters + assignGlobalParameters({"PrintSolutionRejectionReason": True}, cxxCompiler) logicPath = Path(args.LogicPath) pattern = "**/*.yaml" - files = logicPath.glob(pattern) - print(f"Checking logic files with glob {args.LogicPath}{pattern}") + files = list(logicPath.glob(pattern)) - if not any([args.CheckMatrixInstruction]): - print("No checks specified. Exiting.") - exit(0) + batchSize = len(files) // jobs + batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) fn = functools.partial(runChecks, logicPath, gp) - results = ParallelMap2(fn, files, multiArg=False, procs=args.Jobs) - - keep = sum([x[0] for x in results]) - total = sum([x[1] for x in results]) + with Pool(processes=jobs) as pool: + results = pool.map(fn, batches) + + # TIP: This is how to use joblib. Leave for reference. + # for _keep, _total in ParallelMap2( + # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" + # ): + + # keep, total = 0, 0 + for _keep, _total in results: + keep += _keep + total += _total rejects = total - keep print(f"Total {total} solutions") diff --git a/tensilelite/Tensile/TensileLogic/Utilities.py b/tensilelite/Tensile/TensileLogic/Utilities.py new file mode 100644 index 0000000000..4d91143379 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/Utilities.py @@ -0,0 +1,34 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from pathlib import Path +from inspect import currentframe, getframeinfo + + +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index a29b691ec4..3b24578e1e 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -1,8 +1,66 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidMatrixInstruction +--- +Format: (M x N x K x B) + XDLOPS tile definition, only valid for gfx908, gfx90a + MxNxKxB specifies matrix instruction variants + MxNxB determines the shape of the C tile each instruction worked on + K determines the unroll depth + +Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + (Note: MxN means M-by-N in the following comments) + MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + Putting it all together: + - [32, 32, 1, 2, 1, 4, 1, 2, 2] + ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + MatrixInst BlkM WT Wave + - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + +Notes: + - If empty, do not use these instructions +""" + import math from typing import Dict from pathlib import Path from inspect import currentframe, getframeinfo +from Tensile.SolutionStructs import reject +from Tensile.TensileInstructions.DataType import DataType + +from .Utilities import elineno + from Tensile.Common import IsaInfo MI_KEY: str = "MatrixInstruction" @@ -140,97 +198,118 @@ def validateMatrixInstruction(solution: dict, filepath: Path, isaInfoMap: Dict[s AssertionError: If any of the validation checks fail. """ try: - _validateMatrixInstruction(solution, isaInfoMap) + validateMIParameters(solution, isaInfoMap) + assert solution["Valid"], f"Solution was rejected: {elineno()}" return True except AssertionError as e: - print(f"Validation failed: {filepath} (index {solution['SolutionIndex']})") - print(f"Error: file: {e}") + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) return False -def _validateMatrixInstruction(solution: dict, isaInfoMap: Dict[str, IsaInfo]): - """ - Function to validate the matrix instruction for the provided solution. - See exported function for more details. - """ - assert MI_KEY in solution, elineno() - assert MI_ENABLED_KEY in solution, elineno() - assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), elineno() +def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): + assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" + assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" + assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), elineno() + ": MI empty but enabled" isa = tuple(solution["ISA"]) - miFull = solution[MI_KEY] - miEnabled = solution[MI_ENABLED_KEY] - assert miFull in validMatrixInstructions, elineno() + # TODO: Temporary until all 940/941 ISAs are removed + if (9, 4, 0) <= isa <= (9, 4, 1): + isa = (9, 4, 2) - if len(solution[MI_KEY]) == 9: - wfsize = solution["WavefrontSize"] - mi = [miFull[0], miFull[1], miFull[2], miFull[3]] - waves = miFull[7] * miFull[8] - miwg0 = miFull[4] * miFull[0] * miFull[7] # Matrix instruction work group 0 - miwg1 = waves * wfsize // miwg0 + mi4 = solution[MI_KEY] + mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] + assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" + mi9.append(solution["MatrixInstBM"]) + assert "MIWaveTile" in solution, elineno() + ": missing MIWaveTile" + mi9.extend(solution["MIWaveTile"]) + assert "MIWaveGroup" in solution, elineno() + ": missing MIWaveGroup" + mi9.extend(solution["MIWaveGroup"]) + + miEnabled = solution[MI_ENABLED_KEY] - isSparse = solution["ProblemType"]["Sparse"] - miDataType = ( - solution["ProblemType"]["DataType"] - if (not solution["EnableF32XdlMathOp"]) - else solution["ProblemType"]["F32XdlMathOp"] + if len(mi4) == 0: + assert miEnabled == False, elineno() + else: + assert len(mi4) == 4 and len(mi9) == 9, ( + elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) ) - miBlock = solution["MIBlock"] - miWaveGroup = solution["MIWaveGroup"] - miWaveTile = solution["MIWaveTile"] - miInputPerThread = solution["MIInputPerThread"] - miInputPerThreadA = solution["MIInputPerThreadA"] - miInputPerThreadB = solution["MIInputPerThreadB"] - miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] - - # Check work group - assert solution["WorkGroup"] == [miwg0, miwg1], elineno() - - # Check datatype - if not isSparse: - if isaInfoMap[isa].asmCaps["HasMFMA"]: - if not (miDataType.toChar() in validMFMA and mi in validMFMA[miDataType.toChar()]): - assert miDataType.isBFloat16() and mi in validMFMA["B1k"], elineno() - elif isaInfoMap[isa].asmCaps["HasWMMA"]: - assert mi in validWMMA, elineno() - else: - assert miDataType.toChar() in validSMFMA and mi in validSMFMA[miDataType.toChar()], elineno() - - if (not isaInfoMap[isa].asmCaps["HasMFMA"]) and isaInfoMap[isa].asmCaps["HasWMMA"]: - if isa[0] == 10 or isa[0] == 11: - assert miInputPerThread == mi[2], elineno() - - assert solution["MFMA_BF16_1K"] == False, elineno() - - # Check MIBlock - assert miBlock[0] == mi[0], elineno() - assert miBlock[1] == mi[1], elineno() - assert miBlock[2] == mi[2], elineno() - assert miBlock[3] == mi[3], elineno() - assert miBlock[4] == min(miwg0 // mi[0], mi[3]), elineno() - assert miBlock[5] == mi[3] // miBlock[4], elineno() - - # Check MIWaveGroup - assert miWaveGroup[0] == min((miwg0 // mi[0]) // miBlock[4], waves), elineno() - assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() - - # Check MIWaveTile - assert miWaveTile[0] == mi[5], elineno() - assert miWaveTile[1] == mi[6], elineno() - - # Check MIInputPerThread - assert miInputPerThread == mi[0] * mi[2] * mi[3] // wfsize, elineno() - - # TODO: sparsity in hipBLASLt appears to be unused or always zero - sparseA = not isSparse if isSparse != 2 else False - sparseB = isSparse == 2 if isSparse else False - assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() - assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() - assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() - - assert miEnabled == True, elineno() - elif miFull != [] and len(miFull) == 4: - assert miEnabled == True, elineno() + + if not miEnabled: + return + + assert mi4 in validMatrixInstructions, elineno() + + wfsize = solution["WavefrontSize"] + waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] + miwg0 = mi9[4] * mi9[0] * mi9[7] # Matrix instruction work group 0 + miwg1 = waves * wfsize // miwg0 + + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + ptype = solution["ProblemType"] + isSparse = ptype.get("Sparse", 0) + miDataType = DataType( + ptype["DataType"] + if not solution.get("EnableF32XdlMathOp", False) + else ptype["F32XdlMathOp"] + ) + + miBlock = solution["MIBlock"] + miWaveGroup = solution["MIWaveGroup"] + miWaveTile = solution["MIWaveTile"] + + # Check datatype + if not isSparse: + if hasMFMA: + if not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]): + if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: + assert solution["MFMA_BF16_1K"], elineno() + else: + reject(solution, f"Invalid MFMA BFloat16 configuration: {solution}") + elif hasWMMA and (not mi4 in validWMMA): + reject(solution, f"Invalid WMMA configuration: {solution}") else: - assert miEnabled == False, elineno() + if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): + reject(solution, f"Invalid SMFMA configuration: {solution}") + + if (not hasMFMA) and hasWMMA: + if isa[0] == 10 or isa[0] == 11: + assert miInputPerThread == mi4[2], elineno() + + # Check MIBlock + assert miBlock[0] == mi4[0], elineno() + assert miBlock[1] == mi4[1], elineno() + assert miBlock[2] == mi4[2], elineno() + assert miBlock[3] == mi4[3], elineno() + assert miBlock[4] == min(miwg0 // mi4[0], mi4[3]), elineno() + assert miBlock[5] == mi4[3] // miBlock[4], elineno() + + # Check MIWaveGroup + assert miWaveGroup[0] == min((miwg0 // mi4[0]) // miBlock[4], waves), elineno() + assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() + + # Check MIWaveTile + assert miWaveTile[0] == mi9[5], elineno() + assert miWaveTile[1] == mi9[6], elineno() + + # Check MIInputPerThread + miInputPerThread = solution["MIInputPerThread"] + + # If Navi architecture, the input per thread is different + if (10, 0, 0) <= isa <= (11, 0, 2): + assert miInputPerThread == mi4[2], elineno() + else: + assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, elineno() + + # miInputPerThreadA = solution["MIInputPerThreadA"] + # miInputPerThreadB = solution["MIInputPerThreadB"] + # miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] + # sparseA = not isSparse if isSparse != 2 else False + # sparseB = isSparse == 2 if isSparse else False + # assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() + # assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() + # assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() diff --git a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py new file mode 100644 index 0000000000..7de289396d --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py @@ -0,0 +1,58 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidWorkGroup +--- +Dimensions of the workgroup which will operate on a tile and share lds +Example: ( wg0 x wg1 x LocalSplitU ) +""" + +from .Utilities import elineno + +validWorkGroups = [] +for numThreads in range(32, 1025, 32): + for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: + for sg0 in range(1, numThreads // nsg + 1): + sg1 = numThreads // nsg // sg0 + if sg0 * sg1 * nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + + +def validateWorkGroup(solution: dict, globalParams: dict, filepath: str): + try: + _validateWorkGroup(solution, globalParams) + assert solution["Valid"], f"Solution was rejected: {elineno()}" + return True + except AssertionError as e: + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) + return False + + +def _validateWorkGroup(solution: dict, globalParams: dict): + assert "WorkGroup" in solution, elineno() + assert solution["WorkGroup"] in validWorkGroups, elineno() From 140b74e5ab10567808ad886b48726d585b5444a0 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:31:44 -0600 Subject: [PATCH 080/105] refactor: move solution structs to module --- tensilelite/Tensile/BenchmarkProblems.py | 75 ++- tensilelite/Tensile/BenchmarkStructs.py | 10 +- tensilelite/Tensile/ClientWriter.py | 1 + .../Tensile/Common/GlobalParameters.py | 5 +- tensilelite/Tensile/Common/__init__.py | 10 +- tensilelite/Tensile/CustomKernels.py | 2 +- tensilelite/Tensile/KernelWriterAssembly.py | 2 +- tensilelite/Tensile/LibraryIO.py | 16 +- .../Tensile/SolutionStructs/Problem.py | 368 ++++++++++++ .../Solution.py} | 525 ++---------------- .../Tensile/SolutionStructs/Utilities.py | 147 +++++ .../Tensile/SolutionStructs/__init__.py | 3 + tensilelite/Tensile/Tensile.py | 18 +- .../Tensile/TensileCreateLibrary/Run.py | 10 +- tensilelite/Tensile/TensileLogic/Run.py | 78 +-- .../TensileLogic/ValidMatrixInstruction.py | 71 ++- .../Tensile/TensileLogic/ValidWorkGroup.py | 10 +- 17 files changed, 745 insertions(+), 606 deletions(-) create mode 100644 tensilelite/Tensile/SolutionStructs/Problem.py rename tensilelite/Tensile/{SolutionStructs.py => SolutionStructs/Solution.py} (88%) create mode 100644 tensilelite/Tensile/SolutionStructs/Utilities.py create mode 100644 tensilelite/Tensile/SolutionStructs/__init__.py diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index a055b045a1..be8013f9ba 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -34,20 +34,20 @@ from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters +from Tensile.ProblemType import ProblemType +from Tensile.Toolchain.Component import Assembler +from Tensile.SolutionStructs import Solution, ProblemSizes, matrixInstructionToMIParameters from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni from .KernelWriterAssembly import KernelWriterAssembly from .KernelWriter import DebugConfig -from .SolutionStructs import Solution, ProblemSizes -from Tensile.ProblemType import ProblemType from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain -from Tensile.Toolchain.Component import Assembler -from .Common import globalParameters, HR, print1, print2, IsaInfo, \ +from .Common import globalParameters, HR, print1, print2, IsaInfo, defaultSolution, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, DepthUConfig from Tensile.Common.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ @@ -59,23 +59,31 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") - solutions = [] + # solutions = [] solutionSet = set() for perm in forkPermutations: - solution = {"ProblemType": deepcopy(problemType.state)} + # Expect only a single ISA in the map for the Tensile context + # because the GPU has to be physically present for benchmarking + isa = next(iter(isaInfoMap.keys())) + + solution = {} + solution.update({ + "ProblemType": deepcopy(problemType.state), + "ISA": isa, + }) solution.update(constantParams) solution.update(perm) + mi = solution["MatrixInstruction"] - isa = solution["ISA"] wavefrontSize = solution["WavefrontSize"] - enableF32x = solution.get("EnableF32XdlMathOp", False) + workgroup = solution["WorkGroup"] + ptype = solution["ProblemType"] - miParams = Solution.matrixInstructionToMIParameters(mi, isa, wavefrontSize, problemType, enableF32x) + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) solution.update(miParams) - validateMIParameters(solution, globalParameters) - if solution["Valid"]: + if validateMIParameters(solution, isaInfoMap): solutionObject = Solution( solution, debugConfig.splitGSU, @@ -85,17 +93,16 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse assembler, isaInfoMap ) - if solutionObject not in solutionSet: + if solutionObject["Valid"]: solutionSet.add(solutionObject) - solutions.append(solutionObject) elif debugConfig.printSolutionRejectionReason: - print1("rejecting solution " + str(solutionObject)) + print1("rejecting solution " + str(solution)) - return solutions + return list(solutionSet) def _getCustomKernelSolutionObj( - kernelName, + kernelName, internalSupportParams, assembler: Assembler, debugConfig: DebugConfig, @@ -104,9 +111,23 @@ def _getCustomKernelSolutionObj( directory=CUSTOM_KERNEL_PATH ): """Creates the Solution object for a custom kernel""" - config = getCustomKernelConfig(kernelName, internalSupportParams, directory) + sol = getCustomKernelConfig(kernelName, internalSupportParams, directory) + + mi = sol["MatrixInstruction"] + if len(mi) != 9: + printWarning(f"Custom kernel {kernelName} has MI length {len(mi)}, expected 9.") + + isa = next(iter(isaInfoMap.keys())) + + wavefrontSize = sol["WavefrontSize"] + ptype = sol["ProblemType"] + workgroup = sol["WorkGroup"] + + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + sol.update(miParams) + return Solution( - config, + sol, debugConfig.printIndexAssignmentInfo, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, @@ -209,10 +230,10 @@ def writeBenchmarkFiles( kernelSerialNaming = getSerialNaming(kernels) kernelMinNaming = getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, - kernelSerialNaming, - asmToolchain.assembler, - debugConfig, + kernelMinNaming, + kernelSerialNaming, + asmToolchain.assembler, + debugConfig, ) # write solution, kernels and CMake @@ -239,10 +260,10 @@ def writeBenchmarkFiles( newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary( - solutions, - asmToolchain.assembler, - debugConfig.splitGSU, - debugConfig.printSolutionRejectionReason, + solutions, + asmToolchain.assembler, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, depthUConfig, isaInfoMap, @@ -538,7 +559,7 @@ def main( useShortNames, debugConfig, depthUConfig, - deviceId, + deviceId, isaInfoMap ) totalTestFails += benchmarkErrors diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index 55637fb787..849dc5c205 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -24,14 +24,16 @@ from copy import deepcopy import itertools -from .Common import print1, print2, hasParam, printExit, \ + +from Tensile.Common.ValidParameters import checkParametersAreValid +from Tensile.Common import print1, print2, hasParam, printExit, \ defaultBenchmarkCommonParameters, validParameters, globalParameters, \ - defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes, \ - checkParametersAreValid + defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes +from Tensile.ProblemType import ProblemType + from .CustomKernels import getAllCustomKernelNames from .SolutionStructs import ProblemSizes, ActivationArgs, BiasTypeArgs, \ FactorDimArgs -from Tensile.ProblemType import ProblemType def getDefaultsForMissingParameters(paramList, defaultParams): diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 302f66af5c..a940f7b2cc 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -139,6 +139,7 @@ def main(config, assembler: Assembler, cCompiler: str, isaInfoMap, outputPath: P activationArgs = ActivationArgs(problemType, activationEnums) if isForAll else "" factorDimArgs = FactorDimArgs(problemType, factorDimEnums) + print1(f"libraryFile: {yamlList}") clientParametersPaths.append(writeClientConfig( forBenchmark=False, solutions=None, diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 9122e7b83c..ba27cd4dd1 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -480,7 +480,7 @@ def restoreDefaultGlobalParameters(): # hopefully the isaInfoMap keys only contain isas we plan to build and not all -def printCapabilitiesTable(isaInfoMap: Dict[str, IsaInfo]): +def printCapabilitiesTable(isaInfoMap: Dict[str, IsaInfo]): """ Prints a capability table for the given parameters and ISA information map. @@ -502,6 +502,7 @@ def capRow(isaInfoMap, cap, capType): for info in isaInfoMap.values() ] + print1(f"infomap: {isaInfoMap}") gfxs = list(map(isaToGfx, isaInfoMap.keys())) headerRow = ["Capability"] + gfxs @@ -517,7 +518,7 @@ def capRow(isaInfoMap, cap, capType): printTable([headerRow] + asmCapRows + archCapRows) -def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo], cxxCompiler=None): +def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo]): """ Assign Global Parameters Each global parameter has a default parameter, and the user diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index e2dd14fe6a..b49aa2ee79 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -1,12 +1,14 @@ from .Architectures import * from .Capabilities import * from .Constants import * - -# Dunder variables are not exported via `*` from .GlobalParameters import * +# Dunder variables are not exported via `*` from .GlobalParameters import __version__ -#from .Naming import * +# from .Naming import * from .Parallel import * from .Types import * from .Utilities import * -from .ValidParameters import * + +# NOTE: Do not export valid parameters automatically to save memory +# it must be explicitly imported where needed +# from .ValidParameters import * diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index e31a562d3e..bd7a9ba18e 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -23,7 +23,7 @@ ################################################################################ from . import CUSTOM_KERNEL_PATH -from .Common import checkParametersAreValid, validParameters +from .Common.ValidParameters import checkParametersAreValid, validParameters import yaml diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 174df35d59..0d3bebd814 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -48,7 +48,7 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from Tensile.Common import print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection +from Tensile.Common import print1, print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection from Tensile.Common.Naming import getKernelFileBase from Tensile.Toolchain.Component import Assembler diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 4b84fa1a09..2f45dd1c1e 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -321,7 +321,7 @@ def parseLibraryLogicData( """Parses the data of a library logic file.""" if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) - + #is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") #if not (archs is None) and "ArchitectureName" in data: # if isinstance(archs, List): @@ -357,6 +357,10 @@ def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: customConfig = getCustomKernelConfig(solutionState["CustomKernelName"], isp) for key, value in customConfig.items(): solutionState[key] = value + + if len(customConfig["MatrixInstruction"]) != 4: + raise ValueError(f"Custom kernel MatrixInstruction can only be of length 4, found {customConfig['MatrixInstruction']}") + # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -385,13 +389,13 @@ def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: solutions = [solutionStateToSolution(solutionState, assembler, isaInfoMap) for solutionState in data["Solutions"]] newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState( - data, - solutions, - splitGSU, - printSolutionRejectionReason, + data, + solutions, + splitGSU, + printSolutionRejectionReason, printIndexAssignmentInfo, depthUConfig, - assembler, + assembler, isaInfoMap, lazyLibraryLoading ) diff --git a/tensilelite/Tensile/SolutionStructs/Problem.py b/tensilelite/Tensile/SolutionStructs/Problem.py new file mode 100644 index 0000000000..940afe2167 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/Problem.py @@ -0,0 +1,368 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from collections import OrderedDict + +from Tensile.TensileInstructions.Base import fastdeepcopy as deepcopy + + +class ProblemSizeRange: + + def __init__(self, problemType, config): + self.totalIndices = 1+max(problemType["IndexAssignmentsA"]) + problemType["NumIndicesLD"] + if len(config) < self.totalIndices: + for i in range(len(config), self.totalIndices): + if i < self.totalIndices - problemType["NumIndicesLD"]: + config.append(0) + else: + config.append([0]) + + self.indexMax = [] + self.indexIsSized = [] + self.indicesSized = [] + self.indicesMapped = [] + for i in range(0, self.totalIndices): + dim = deepcopy(config[i]) + if isinstance(dim, list): + if len(dim) == 1: + self.indicesSized.append([dim[0], 1, 0, dim[0]]) + elif len(dim) == 2: + self.indicesSized.append([dim[0], dim[0], 0, dim[1]]) + elif len(dim) == 3: + self.indicesSized.append([dim[0], dim[1], 0, dim[2]]) + elif len(dim) == 4: + self.indicesSized.append([dim[0], dim[1], dim[2], dim[3]]) + else: + printExit("dimension[%u] config (%s) has %u descriptors rather than 1-4." + % ( i, dim, len(dim) )) + self.indexIsSized.append(True) + self.indexMax.append(self.indicesSized[len(self.indicesSized)-1][3]) + + elif isinstance(dim, int): + self.indicesMapped.append(dim) + self.indexIsSized.append(False) + self.indexMax.append(self.indicesSized[self.indicesMapped[ \ + len(self.indicesMapped)-1]][3]) + + # max num elements in each tensor + self.maxNumElements = [ 1, 1, 1 ] + for i in range(0, problemType["NumIndicesC"]): + self.maxNumElements[0] *= self.indexMax[i] + for i in problemType["IndexAssignmentsA"]: + self.maxNumElements[1] *= self.indexMax[i] + for i in problemType["IndexAssignmentsB"]: + self.maxNumElements[2] *= self.indexMax[i] + + self.totalProblemSizes = 1 + self.numProblemSizes = [] # per index + self.problemSizeToIndex = [] + self.problemIndexToSize = [] + sizedIdx = 0 + for i in range(0, len(self.indexIsSized)): + self.problemSizeToIndex.append({}) + self.problemIndexToSize.append({}) + if self.indexIsSized[i]: + self.numProblemSizes.append(0) + index = self.indicesSized[sizedIdx] + sizedIdx += 1 + currentSize = index[0] + currentIncrement = index[1] + while currentSize <= index[3]: + currentSize += currentIncrement + currentIncrement += index[2] + self.numProblemSizes[i] += 1 + else: + self.numProblemSizes.append(1) + self.totalProblemSizes *= self.numProblemSizes[i] + + ######################################## + # enumerate problem sizes + currentSizedIndexSizes = [] + currentSizedIndexIncrements = [] + for i in range(0, len(self.indicesSized)): + currentSizedIndexSizes.append(self.indicesSized[i][0]) + currentSizedIndexIncrements.append(self.indicesSized[i][1]) + + # iterate over all problem sizes + self.problemSizes = [] + moreProblemSizes = True + problemIdx = 0 + problemSize = [0]*self.totalIndices + while moreProblemSizes: + #/ convert current sized and mapped indices to full sizes + currentSizedIdx = 0 + currentMappedIdx = 0 + for i in range(0, self.totalIndices): + if self.indexIsSized[i]: + problemSize[i] = currentSizedIndexSizes[currentSizedIdx] + currentSizedIdx+=1 + else: + problemSize[i] = problemSize[self.indicesMapped[currentMappedIdx]] + currentMappedIdx+=1 + self.problemSizes.append(tuple(problemSize)) + + #/ increment sizes for next benchmark + currentSizedIndexSizes[0] += currentSizedIndexIncrements[0] + currentSizedIndexIncrements[0] += self.indicesSized[0][2] + for i in range(1, len(self.indicesSized)+1): + # if prior index past max, reset to min and increment next index + if currentSizedIndexSizes[i-1] > self.indicesSized[i-1][3]: + #/ reset prior index + currentSizedIndexSizes[i-1] = self.indicesSized[i-1][0] + currentSizedIndexIncrements[i-1] = self.indicesSized[i-1][1] + # increment next index + if i >= len(self.indicesSized): + moreProblemSizes = False + else: + currentSizedIndexSizes[i] += currentSizedIndexIncrements[i] + currentSizedIndexIncrements[i] += self.indicesSized[i][2] + + problemIdx+=1 + + ######################################## + # YAML format + def __str__(self): + state = "[ " + sizedIdx = 0 + mappedIdx = 0 + for i in range(0, len(self.indexIsSized)): + if self.indexIsSized[i]: + indices = self.indicesSized[sizedIdx] + state += "[ %u, %u, %u, %u ]" \ + % (indices[0], indices[1], indices[2], indices[3]) + sizedIdx += 1 + else: + indices = self.indicesSized[self.indicesMapped[mappedIdx]] + state += str(self.indicesMapped[mappedIdx]) + mappedIdx += 1 + if i < len(self.indexIsSized)-1: + state += ", " + state += " ]" + return state + +class Problem: + """ Problem sizes, strides, padding and other info""" + def __init__(self, sizes=None, stridesA=None, stridesB=None, stridesC=None, stridesD=None, count=None): + self.sizes = tuple(sizes) if sizes else None + self.stridesA = tuple(stridesA) if stridesA else None + self.stridesB = tuple(stridesB) if stridesB else None + self.stridesC = tuple(stridesC) if stridesC else None + self.stridesD = tuple(stridesD) if stridesD else None + + self.count = count + + def __str__(self): + rv= "{ sizes:" + str(list(self.sizes)) + if self.stridesA: + rv += ", stridesA:" + str(list(self.stridesA)) + if self.stridesB: + rv += ", stridesB:" + str(list(self.stridesB)) + if self.stridesC: + rv += ", stridesC:" + str(list(self.stridesC)) + if self.stridesD: + rv += ", stridesD:" + str(list(self.stridesD)) + rv += " }" + return rv + +class ExactList(Problem): + def __init__(self, e, problemType): + if len(e) == problemType["TotalIndices"]: + if -1 in e: + printExit("ExactSize %s contains -1" % (e)) + if problemType["OperationType"] == "GEMM": + e += [-1, -1, -1, -1] + e = ExactList.convertLeadingDims(problemType, tuple(e)) + sizes=e + + elif len(e) == (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + sizes = ExactList.convertLeadingDims(problemType, tuple(e)) + else: + printExit("ExactSize %s doesn't match indices of ProblemType %s, totalIndices=%d, len e=%d, NumIndicesLD = %d" \ + % (e, problemType, problemType["TotalIndices"], len(e), problemType["NumIndicesLD"]) ) + + # TODO- pass strides here, remove calls to convertLeadingDims + Problem.__init__(self, sizes=sizes) + + def __str__(self): + return str(list(self.sizes)) + + @staticmethod + def convertLeadingDims(problemType, problemSize, stridesA = None, stridesB = None, stridesC = None, stridesD = None): + # FIXME-problem: refactor to eliminate max, pass strides in strideB parm rather than hacked + # onto the end of the sizes list + predStridesD = stridesD is not None and stridesD[1] != -1 + predStridesC = stridesC is not None and stridesC[1] != -1 + predStridesA = stridesA is not None and stridesA[1] != -1 + predStridesB = stridesB is not None and stridesB[1] != -1 + return problemSize[:problemType["NumIndicesC"]+1] + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][0]]) if not predStridesD else stridesD[1], ) + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][1]]) if not predStridesC else stridesC[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][2]], + problemSize[problemType["IndexAssignmentsA"][0]]) if not predStridesA else stridesA[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][3]], + problemSize[problemType["IndexAssignmentsB"][0]]) if not predStridesB else stridesB[1], ) + + +class ExactDict(Problem): + AllowedFields = [ 'count', 'sizes', 'stridesA', 'stridesB', 'stridesC', 'stridesD' ] + + def __init__(self, e, problemType): + Problem.__init__(self) + + for f in e: + if f in ExactDict.AllowedFields: + setattr(self, f, e[f]) + else: + raise RuntimeError ("specified field '%s' is not a valid Exact dict field"%f) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + sizesTuple = tuple(self.sizes + [-1, -1, -1, -1]) + self.sizes = ExactList.convertLeadingDims(problemType, sizesTuple, self.stridesA, self.stridesB, self.stridesC, self.stridesD) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + if len(self.sizes) != (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + # FIXME-ExactDict size descriptor still (but preferrably not so) uses 8-tuple for GEMM problems + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"]+problemType["NumIndicesLD"], len(self.sizes))) + elif len(self.sizes) != problemType["TotalIndices"]: + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"], len(self.sizes))) + + +################################################################################ +# ProblemSizes +################################################################################ +""" +Adapter class for class `ProblemSizes`. It satisfies the implicit usage requirement +of ClientWriter.writeClientConfig() by converting ExactLogic to list of `Problem` objects +""" +class ProblemSizesMock: + def __init__(self, exactLogic): + self.problems = [Problem(problem) for problem, solution in exactLogic] + +class ProblemSizesMockDummy: + def __init__(self): + self.problems = [Problem(sizes=[128, 128, 1, 512])] + +class ProblemSizes: + + ######################################## + def __init__(self, problemType, config): + self.problemType = problemType + self.ranges = [] + self.exacts = [] + self.minStrides = None + if config: + for dictionary in config: + for sizeTypeKey in dictionary: + #print ("PROBLEM parsed:", sizeTypeKey, dictionary[sizeTypeKey]) + if sizeTypeKey == "Range": + psr = ProblemSizeRange(problemType, dictionary[sizeTypeKey]) + self.ranges.append( psr ) + elif sizeTypeKey == "Exact": + e= dictionary[sizeTypeKey] + if isinstance(e,list): + self.exacts.append(ExactList(e, problemType)) + elif isinstance(e,dict): + self.exacts.append(ExactDict(e, problemType)) + else: + printExit("Unsupported Exact type==%s"%type(e)) + elif sizeTypeKey == "MinStride": + e = dictionary[sizeTypeKey] + if len(e) != problemType["TotalIndices"]: + printExit("MinStride %s doesn't match indices of ProblemType %s" \ + % (e, problemType) ) + if self.minStrides: + printExit("Only one MinStride command is allowed in a ProblemsSizes definition. Previous minStrides:%s, New minstride:%s" \ + % (self.minStrides, e) ) + + self.minStrides=(tuple(e)) + else: + printExit("ProblemSize Type %s not supported"%sizeTypeKey) + + if not self.minStrides: + # set harmless default mins of 0 + self.minStrides = ([0]* problemType["TotalIndices"]) + + # not the ideal spot, but convert leading dims that are below the minimum size + if problemType["OperationType"] == "GEMM": + for i in range(0, len(self.ranges)): + self.ranges[i].problemSizes[:] = \ + [ExactList.convertLeadingDims(self.problemType, problemSize) for problemSize in self.ranges[i].problemSizes] + + self.problems = OrderedDict() + for sizeRange in self.ranges: + for rangeSize in sizeRange.problemSizes: + self.problems.update({Problem(rangeSize) : 1}) + for e in self.exacts: + self.problems.update({e : 1}) + self.problems = list(self.problems.keys()) + self.totalProblemSizes = len(self.problems) + + # max sizes + self.maxD = 0 + self.maxC = 0 + self.maxA = 0 + self.maxB = 0 + for problem in self.problems: + problemSize = problem.sizes # FIXME-problem. This should use problem.strides* + + sizeLdd = problemSize[self.problemType["IndexAssignmentsLD"][0]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeD = max(self.minStrides[0], sizeLdd) + for i in range(1, problemType["NumIndicesC"]): + sizeD *= max(self.minStrides[i], problemSize[i]) + + sizeLdc = problemSize[self.problemType["IndexAssignmentsLD"][1]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeC = max(self.minStrides[0], sizeLdc) + for i in range(1, problemType["NumIndicesC"]): + sizeC *= max(self.minStrides[i], problemSize[i]) + + sizeLda = problemSize[self.problemType["IndexAssignmentsLD"][2]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsA"][0]] + sizeA = max(self.minStrides[self.problemType["IndexAssignmentsA"][0]], sizeLda) + for i in self.problemType["IndexAssignmentsA"][1:]: + sizeA *= max(self.minStrides[i], problemSize[i]) + + sizeLdb = problemSize[self.problemType["IndexAssignmentsLD"][3]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsB"][0]] + sizeB = max(self.minStrides[self.problemType["IndexAssignmentsB"][0]], sizeLdb) + for i in self.problemType["IndexAssignmentsB"][1:]: + sizeB *= max(self.minStrides[i], problemSize[i]) + + self.maxD = max(self.maxD, sizeD) + self.maxC = max(self.maxC, sizeC) + self.maxA = max(self.maxA, sizeA) + self.maxB = max(self.maxB, sizeB) + + def __str__(self): + s = "ProblemSizes\n" + for sizeRange in self.ranges: + s += " %s" % sizeRange + return s + + diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs/Solution.py similarity index 88% rename from tensilelite/Tensile/SolutionStructs.py rename to tensilelite/Tensile/SolutionStructs/Solution.py index a693f0eb7c..63f996af1e 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -22,33 +22,33 @@ # ################################################################################ -from .TensileInstructions import DataType, roundUpToNearestMultiple -from .TensileInstructions.Base import fastdeepcopy as deepcopy +from Tensile.TensileInstructions import DataType, roundUpToNearestMultiple +from Tensile.TensileInstructions.Base import fastdeepcopy as deepcopy -from .KernelWriterBetaOnly import KernelWriterBetaOnly -from .KernelWriterConversion import KernelWriterConversion -from .KernelWriterActivationEnumHeader import KernelWriterActivationEnumHeader -from .KernelWriterActivationFunction import KernelWriterActivationFunction -from .KernelWriterActivationOnly import KernelWriterActivationOnly -from .KernelWriterReduction import KernelWriterReduction +from Tensile.KernelWriterBetaOnly import KernelWriterBetaOnly +from Tensile.KernelWriterConversion import KernelWriterConversion +from Tensile.KernelWriterActivationEnumHeader import KernelWriterActivationEnumHeader +from Tensile.KernelWriterActivationFunction import KernelWriterActivationFunction +from Tensile.KernelWriterActivationOnly import KernelWriterActivationOnly +from Tensile.KernelWriterReduction import KernelWriterReduction -from .Activation import ActivationType -from .AsmStoreState import VectorDataTypes +from Tensile.Activation import ActivationType +from Tensile.AsmStoreState import VectorDataTypes -from .CustomKernels import isCustomKernelConfig -from .Common import assignParameterWithDefault, IsaInfo, \ +from Tensile.CustomKernels import isCustomKernelConfig +from Tensile.Common import assignParameterWithDefault, IsaInfo, \ defaultSolution, \ defaultInternalSupportParams, \ internalParameters, \ - print2, printExit, printWarning, \ - validMFMA, validSMFMA, \ - roundUp, validWMMA, \ - INDEX_CHARS, IsaVersion, SemanticVersion, \ + print1, print2, printExit, printWarning, \ + roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ DepthUConfig from Tensile.Common.Naming import getNameFull from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler +from .Utilities import reject + from collections import OrderedDict from collections.abc import Mapping @@ -59,32 +59,7 @@ import math import sys -count = 0 - - -######################################## -# Print a reject message : -def reject(state, printSolutionRejectionReason: bool, *args): - if state and "NoReject" in state and state["NoReject"]: - return - if printSolutionRejectionReason: - sys.stdout.write("\nreject: ") - for a in args: - print(a) - #traceback.print_stack(None, 2) - solutionIndex = state["SolutionIndex"] if (state != None and "SolutionIndex" in state) else -1 - if solutionIndex != -1: - # If we have valid solutionIndex, this means we are during TensileCreateLibrary stage - # In this stage, all solutions in the logic should be valid - # So if any rejection happens, print the warning for further check - # This will be done only when --global-parameters=PrintSolutionRejectionReason=True - solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None - # if we don't have SolutionNameMin, we simply use the problemTypeName - solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin - raise Exception("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ - SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) - if state != None: - state["Valid"] = False + # print a labled variable def pvar(state, field): @@ -99,352 +74,9 @@ class Fbs(Enum): Sum=2 # Expect to be summation dimension -################################################################################ -# ProblemSizeRange -################################################################################ -class ProblemSizeRange: - - ######################################## - def __init__(self, problemType, config): - self.totalIndices = 1+max(problemType["IndexAssignmentsA"]) + problemType["NumIndicesLD"] - if len(config) < self.totalIndices: - for i in range(len(config), self.totalIndices): - if i < self.totalIndices - problemType["NumIndicesLD"]: - config.append(0) - else: - config.append([0]) - - self.indexMax = [] - self.indexIsSized = [] - self.indicesSized = [] - self.indicesMapped = [] - for i in range(0, self.totalIndices): - dim = deepcopy(config[i]) - if isinstance(dim, list): - if len(dim) == 1: - self.indicesSized.append([dim[0], 1, 0, dim[0]]) - elif len(dim) == 2: - self.indicesSized.append([dim[0], dim[0], 0, dim[1]]) - elif len(dim) == 3: - self.indicesSized.append([dim[0], dim[1], 0, dim[2]]) - elif len(dim) == 4: - self.indicesSized.append([dim[0], dim[1], dim[2], dim[3]]) - else: - printExit("dimension[%u] config (%s) has %u descriptors rather than 1-4." - % ( i, dim, len(dim) )) - self.indexIsSized.append(True) - self.indexMax.append(self.indicesSized[len(self.indicesSized)-1][3]) - - elif isinstance(dim, int): - self.indicesMapped.append(dim) - self.indexIsSized.append(False) - self.indexMax.append(self.indicesSized[self.indicesMapped[ \ - len(self.indicesMapped)-1]][3]) - - # max num elements in each tensor - self.maxNumElements = [ 1, 1, 1 ] - for i in range(0, problemType["NumIndicesC"]): - self.maxNumElements[0] *= self.indexMax[i] - for i in problemType["IndexAssignmentsA"]: - self.maxNumElements[1] *= self.indexMax[i] - for i in problemType["IndexAssignmentsB"]: - self.maxNumElements[2] *= self.indexMax[i] - - self.totalProblemSizes = 1 - self.numProblemSizes = [] # per index - self.problemSizeToIndex = [] - self.problemIndexToSize = [] - sizedIdx = 0 - for i in range(0, len(self.indexIsSized)): - self.problemSizeToIndex.append({}) - self.problemIndexToSize.append({}) - if self.indexIsSized[i]: - self.numProblemSizes.append(0) - index = self.indicesSized[sizedIdx] - sizedIdx += 1 - currentSize = index[0] - currentIncrement = index[1] - while currentSize <= index[3]: - currentSize += currentIncrement - currentIncrement += index[2] - self.numProblemSizes[i] += 1 - else: - self.numProblemSizes.append(1) - self.totalProblemSizes *= self.numProblemSizes[i] - - ######################################## - # enumerate problem sizes - currentSizedIndexSizes = [] - currentSizedIndexIncrements = [] - for i in range(0, len(self.indicesSized)): - currentSizedIndexSizes.append(self.indicesSized[i][0]) - currentSizedIndexIncrements.append(self.indicesSized[i][1]) - - # iterate over all problem sizes - self.problemSizes = [] - moreProblemSizes = True - problemIdx = 0 - problemSize = [0]*self.totalIndices - while moreProblemSizes: - #/ convert current sized and mapped indices to full sizes - currentSizedIdx = 0 - currentMappedIdx = 0 - for i in range(0, self.totalIndices): - if self.indexIsSized[i]: - problemSize[i] = currentSizedIndexSizes[currentSizedIdx] - currentSizedIdx+=1 - else: - problemSize[i] = problemSize[self.indicesMapped[currentMappedIdx]] - currentMappedIdx+=1 - self.problemSizes.append(tuple(problemSize)) - - #/ increment sizes for next benchmark - currentSizedIndexSizes[0] += currentSizedIndexIncrements[0] - currentSizedIndexIncrements[0] += self.indicesSized[0][2] - for i in range(1, len(self.indicesSized)+1): - # if prior index past max, reset to min and increment next index - if currentSizedIndexSizes[i-1] > self.indicesSized[i-1][3]: - #/ reset prior index - currentSizedIndexSizes[i-1] = self.indicesSized[i-1][0] - currentSizedIndexIncrements[i-1] = self.indicesSized[i-1][1] - # increment next index - if i >= len(self.indicesSized): - moreProblemSizes = False - else: - currentSizedIndexSizes[i] += currentSizedIndexIncrements[i] - currentSizedIndexIncrements[i] += self.indicesSized[i][2] - - problemIdx+=1 - - ######################################## - # YAML format - def __str__(self): - state = "[ " - sizedIdx = 0 - mappedIdx = 0 - for i in range(0, len(self.indexIsSized)): - if self.indexIsSized[i]: - indices = self.indicesSized[sizedIdx] - state += "[ %u, %u, %u, %u ]" \ - % (indices[0], indices[1], indices[2], indices[3]) - sizedIdx += 1 - else: - indices = self.indicesSized[self.indicesMapped[mappedIdx]] - state += str(self.indicesMapped[mappedIdx]) - mappedIdx += 1 - if i < len(self.indexIsSized)-1: - state += ", " - state += " ]" - return state - -class Problem: - """ Problem sizes, strides, padding and other info""" - def __init__(self, sizes=None, stridesA=None, stridesB=None, stridesC=None, stridesD=None, count=None): - self.sizes = tuple(sizes) if sizes else None - self.stridesA = tuple(stridesA) if stridesA else None - self.stridesB = tuple(stridesB) if stridesB else None - self.stridesC = tuple(stridesC) if stridesC else None - self.stridesD = tuple(stridesD) if stridesD else None - - self.count = count - - def __str__(self): - rv= "{ sizes:" + str(list(self.sizes)) - if self.stridesA: - rv += ", stridesA:" + str(list(self.stridesA)) - if self.stridesB: - rv += ", stridesB:" + str(list(self.stridesB)) - if self.stridesC: - rv += ", stridesC:" + str(list(self.stridesC)) - if self.stridesD: - rv += ", stridesD:" + str(list(self.stridesD)) - rv += " }" - return rv - -class ExactList(Problem): - def __init__(self, e, problemType): - if len(e) == problemType["TotalIndices"]: - if -1 in e: - printExit("ExactSize %s contains -1" % (e)) - if problemType["OperationType"] == "GEMM": - e += [-1, -1, -1, -1] - e = ExactList.convertLeadingDims(problemType, tuple(e)) - sizes=e - - elif len(e) == (problemType["TotalIndices"] + problemType["NumIndicesLD"]): - sizes = ExactList.convertLeadingDims(problemType, tuple(e)) - else: - printExit("ExactSize %s doesn't match indices of ProblemType %s, totalIndices=%d, len e=%d, NumIndicesLD = %d" \ - % (e, problemType, problemType["TotalIndices"], len(e), problemType["NumIndicesLD"]) ) - - # TODO- pass strides here, remove calls to convertLeadingDims - Problem.__init__(self, sizes=sizes) - - def __str__(self): - return str(list(self.sizes)) - - @staticmethod - def convertLeadingDims(problemType, problemSize, stridesA = None, stridesB = None, stridesC = None, stridesD = None): - # FIXME-problem: refactor to eliminate max, pass strides in strideB parm rather than hacked - # onto the end of the sizes list - predStridesD = stridesD is not None and stridesD[1] != -1 - predStridesC = stridesC is not None and stridesC[1] != -1 - predStridesA = stridesA is not None and stridesA[1] != -1 - predStridesB = stridesB is not None and stridesB[1] != -1 - return problemSize[:problemType["NumIndicesC"]+1] + \ - (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][0]]) if not predStridesD else stridesD[1], ) + \ - (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][1]]) if not predStridesC else stridesC[1], ) + \ - (max(problemSize[problemType["IndexAssignmentsLD"][2]], - problemSize[problemType["IndexAssignmentsA"][0]]) if not predStridesA else stridesA[1], ) + \ - (max(problemSize[problemType["IndexAssignmentsLD"][3]], - problemSize[problemType["IndexAssignmentsB"][0]]) if not predStridesB else stridesB[1], ) - - -class ExactDict(Problem): - AllowedFields = [ 'count', 'sizes', 'stridesA', 'stridesB', 'stridesC', 'stridesD' ] - - def __init__(self, e, problemType): - Problem.__init__(self) - - for f in e: - if f in ExactDict.AllowedFields: - setattr(self, f, e[f]) - else: - raise RuntimeError ("specified field '%s' is not a valid Exact dict field"%f) - - if problemType: - if "OperationType" in problemType and problemType["OperationType"] == "GEMM": - sizesTuple = tuple(self.sizes + [-1, -1, -1, -1]) - self.sizes = ExactList.convertLeadingDims(problemType, sizesTuple, self.stridesA, self.stridesB, self.stridesC, self.stridesD) - - if problemType: - if "OperationType" in problemType and problemType["OperationType"] == "GEMM": - if len(self.sizes) != (problemType["TotalIndices"] + problemType["NumIndicesLD"]): - # FIXME-ExactDict size descriptor still (but preferrably not so) uses 8-tuple for GEMM problems - raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ - % (self.sizes, problemType["TotalIndices"]+problemType["NumIndicesLD"], len(self.sizes))) - elif len(self.sizes) != problemType["TotalIndices"]: - raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ - % (self.sizes, problemType["TotalIndices"], len(self.sizes))) - - -################################################################################ -# ProblemSizes -################################################################################ -""" -Adapter class for class `ProblemSizes`. It satisfies the implicit usage requirement -of ClientWriter.writeClientConfig() by converting ExactLogic to list of `Problem` objects -""" -class ProblemSizesMock: - def __init__(self, exactLogic): - self.problems = [Problem(problem) for problem, solution in exactLogic] - -class ProblemSizesMockDummy: - def __init__(self): - self.problems = [Problem(sizes=[128, 128, 1, 512])] - -class ProblemSizes: - - ######################################## - def __init__(self, problemType, config): - self.problemType = problemType - self.ranges = [] - self.exacts = [] - self.minStrides = None - if config: - for dictionary in config: - for sizeTypeKey in dictionary: - #print ("PROBLEM parsed:", sizeTypeKey, dictionary[sizeTypeKey]) - if sizeTypeKey == "Range": - psr = ProblemSizeRange(problemType, dictionary[sizeTypeKey]) - self.ranges.append( psr ) - elif sizeTypeKey == "Exact": - e= dictionary[sizeTypeKey] - if isinstance(e,list): - self.exacts.append(ExactList(e, problemType)) - elif isinstance(e,dict): - self.exacts.append(ExactDict(e, problemType)) - else: - printExit("Unsupported Exact type==%s"%type(e)) - elif sizeTypeKey == "MinStride": - e = dictionary[sizeTypeKey] - if len(e) != problemType["TotalIndices"]: - printExit("MinStride %s doesn't match indices of ProblemType %s" \ - % (e, problemType) ) - if self.minStrides: - printExit("Only one MinStride command is allowed in a ProblemsSizes definition. Previous minStrides:%s, New minstride:%s" \ - % (self.minStrides, e) ) - - self.minStrides=(tuple(e)) - else: - printExit("ProblemSize Type %s not supported"%sizeTypeKey) - - if not self.minStrides: - # set harmless default mins of 0 - self.minStrides = ([0]* problemType["TotalIndices"]) - - # not the ideal spot, but convert leading dims that are below the minimum size - if problemType["OperationType"] == "GEMM": - for i in range(0, len(self.ranges)): - self.ranges[i].problemSizes[:] = \ - [ExactList.convertLeadingDims(self.problemType, problemSize) for problemSize in self.ranges[i].problemSizes] - - self.problems = OrderedDict() - for sizeRange in self.ranges: - for rangeSize in sizeRange.problemSizes: - self.problems.update({Problem(rangeSize) : 1}) - for e in self.exacts: - self.problems.update({e : 1}) - self.problems = list(self.problems.keys()) - self.totalProblemSizes = len(self.problems) - - # max sizes - self.maxD = 0 - self.maxC = 0 - self.maxA = 0 - self.maxB = 0 - for problem in self.problems: - problemSize = problem.sizes # FIXME-problem. This should use problem.strides* - - sizeLdd = problemSize[self.problemType["IndexAssignmentsLD"][0]] if problemType["OperationType"] == "GEMM" else problemSize[0] - sizeD = max(self.minStrides[0], sizeLdd) - for i in range(1, problemType["NumIndicesC"]): - sizeD *= max(self.minStrides[i], problemSize[i]) - - sizeLdc = problemSize[self.problemType["IndexAssignmentsLD"][1]] if problemType["OperationType"] == "GEMM" else problemSize[0] - sizeC = max(self.minStrides[0], sizeLdc) - for i in range(1, problemType["NumIndicesC"]): - sizeC *= max(self.minStrides[i], problemSize[i]) - - sizeLda = problemSize[self.problemType["IndexAssignmentsLD"][2]] \ - if problemType["OperationType"] == "GEMM" \ - else problemSize[self.problemType["IndexAssignmentsA"][0]] - sizeA = max(self.minStrides[self.problemType["IndexAssignmentsA"][0]], sizeLda) - for i in self.problemType["IndexAssignmentsA"][1:]: - sizeA *= max(self.minStrides[i], problemSize[i]) - - sizeLdb = problemSize[self.problemType["IndexAssignmentsLD"][3]] \ - if problemType["OperationType"] == "GEMM" \ - else problemSize[self.problemType["IndexAssignmentsB"][0]] - sizeB = max(self.minStrides[self.problemType["IndexAssignmentsB"][0]], sizeLdb) - for i in self.problemType["IndexAssignmentsB"][1:]: - sizeB *= max(self.minStrides[i], problemSize[i]) - - self.maxD = max(self.maxD, sizeD) - self.maxC = max(self.maxC, sizeC) - self.maxA = max(self.maxA, sizeA) - self.maxB = max(self.maxB, sizeB) - - def __str__(self): - s = "ProblemSizes\n" - for sizeRange in self.ranges: - s += " %s" % sizeRange - return s - ################################################################################ # Factor Type ################################################################################ - class FactorDimArgs: ######################################## @@ -576,16 +208,19 @@ def __init__( else: self["InternalSupportParams"] = defaultInternalSupportParams - # assign parameters with defaults - for key in defaultSolution: - assignParameterWithDefault(self._state, key, config, defaultSolution) + # # assign parameters with defaults + # for key in defaultSolution: + # assignParameterWithDefault(self._state, key, config, defaultSolution) + if 'ISA' not in self._state: if 'ISA' in config: + # The ISA is expected to be defined when calling from TensileCreateLibrary isa = config['ISA'] isa = IsaVersion(isa[0], isa[1], isa[2]) assert self.isaInfoMap[isa].asmCaps["SupportedISA"] self._state['ISA'] = IsaVersion(isa[0], isa[1], isa[2]) else: + # When calling from Tensile, the ISA is typically not defined. printWarning(f"ISA not set on config using {targetIsas[0]}.") self._state['ISA'] = targetIsas[0] @@ -822,6 +457,8 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): reject(state, printRejectionReason, "General Batched GEMM only support GEMM OperationType") + ### ---> This is where we previously called matrixInstructionToMIParameters + EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None if EnableMatrixInstruction == None: if ("MIBlock" in state and len(state["MIBlock"]) == 6) \ @@ -1025,84 +662,6 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota return True - @staticmethod - def matrixInstructionToMIParameters( - mi: list, - isa: IsaVersion, - wavefrontSize: int, - problemType: dict, - enableF32x: bool, - isaInfoMap: Dict[str, IsaInfo] - ): - """ - Converts a 9-item matrix instruction into the associated 4-item alternative, along with - supporting MI parameters. - - Args: - mi: The matrix instruction to convert. Must have length 9. - isa: The ISA tuple. - wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. - problemType: The problem type dictionary. Typically "ProblemType" in a solution. - enableF32x: Whether to enable F32x. Typically "EnableF32XdlMathOp" in a solution. - """ - if len(mi) != 9: - raise ValueError(f"MatrixInstruction must be 9 items long to convert into MI" - f" Parameters, found {mi} with length {len(mi)}") - - result = {} - - mi4 = [mi[0], mi[1], mi[2], mi[3]] - result["MatrixInstruction"] = mi4 - result["EnableMatrixInstruction"] = True - - waves = mi[7]* mi[8] - miwg0 = mi[4] * mi[0] * mi[7] - - result["WorkGroup"] = [miwg0, waves*wavefrontSize // miwg0] - result["ThreadTile"] = [1, 1] - - isSparse = problemType.get(["Sparse"], 0) - miDataType = DataType( - problemType["DataType"] - if not enableF32x - else problemType["F32XdlMathOp"] - ) - - result["MFMA_BF16_1K"] = ( - not isSparse - and isaInfoMap[isa].asmCaps["HasMFMA"] - and not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]) - and miDataType.isBFloat16() - and mi4 in validMFMA["B1k"] - ) - - # set MIBlock - MIBlockBM = miwg0 // mi[0] - MIBlockBM = min(MIBlockBM, mi[3]) - MIBlockBN = mi[3] // MIBlockBM - result["MIBlock"] = [mi[0], mi[1], mi[2], mi[3], MIBlockBM, MIBlockBN] - miwg0 = min((miwg0 // mi[0]) // MIBlockBM, waves) - - # set MIWaveGroup - result['MIWaveGroup'][0] = min((miwg0 // mi[0]) // MIBlockBM, waves) - result['MIWaveGroup'][1] = waves // result['MIWaveGroup'][0] - - # set MIWaveTile - result['MIWaveTile'] = [mi[5], mi[6]] - - # set MIInputPerThread - hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] - hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] - - result['MIInputPerThread'] = mi[0] * mi[2] * mi[3] // wavefrontSize - if (not hasMFMA) and hasWMMA and (isa[0] == 10 or isa[0] == 11): - result['MIInputPerThread'] = mi[2] - sparseA = False if not isSparse or isSparse == 2 else True - sparseB = True if isSparse == 2 else False - result['MIInputPerThreadA'] = result['MIInputPerThread'] if not sparseA else result['MIInputPerThread']//2 - result['MIInputPerThreadB'] = result['MIInputPerThread'] if not sparseB else result['MIInputPerThread']//2 - result['MIInputPerThreadMetadata'] = result['MIInputPerThread'] if not result["ProblemType"]["Sparse"] else result['MIInputPerThread']//8 - return result ############################################## @@ -1449,12 +1008,12 @@ def assignDerivedParameters( rocmVersion: SemanticVersion, depthUConfig: DepthUConfig ): - state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. - #enable F32 xDL MathOp only when the input type is f32. - if "F32XdlMathOp" in state["ProblemType"] \ - and (not state["ProblemType"]["F32XdlMathOp"].isSingle()) \ - and (state["ProblemType"]["DataType"].isSingle()): - state["EnableF32XdlMathOp"] = True + # state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. + # #enable F32 xDL MathOp only when the input type is f32. + # if "F32XdlMathOp" in state["ProblemType"] \ + # and (not state["ProblemType"]["F32XdlMathOp"].isSingle()) \ + # and (state["ProblemType"]["DataType"].isSingle()): + # state["EnableF32XdlMathOp"] = True Solution.assignProblemIndependentDerivedParameters(state, printRejectionReason, isaInfoMap) @@ -1879,16 +1438,16 @@ def assignDerivedParameters( state["ValidDepthU"] = True state["DepthU"] = depthuList[index[0]] Solution.depthUIteration( - state, - index, - depthuList, - problemType, - isa, - bufferLoad, - packedC0, - packedC1, - printRejectionReason, - isaInfoMap, + state, + index, + depthuList, + problemType, + isa, + bufferLoad, + packedC0, + packedC1, + printRejectionReason, + isaInfoMap, rocmVersion, depthUConfig, ) @@ -3548,7 +3107,7 @@ def calcEpilogueTurns(factorDims: List) -> int: # if state["GlobalSplitU"] > 1: # if state["ProblemType"]["SupportUserArgs"] and state["_GlobalAccumulation"] != 'MultipleBufferSingleKernel': # reject(state, printRejectionReason, "Currently SupportUserArgs does not support GSU > 1.") - + if state["_GlobalAccumulation"] == 'MultipleBufferSingleKernel': if state["NumElementsPerBatchStore"] == 1: reject(state, printRejectionReason, "too many store at MultipleBufferSingleKernel direct reject") diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py new file mode 100644 index 0000000000..f88597ead5 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -0,0 +1,147 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import sys +import pprint +from typing import Dict + +from Tensile.Common import IsaVersion, IsaInfo, print1 +from Tensile.Common.ValidParameters import validMFMA +from Tensile.TensileInstructions import DataType + +def reject(state, printSolutionRejectionReason: bool = True, *args): + if state and "NoReject" in state and state["NoReject"]: + return + if printSolutionRejectionReason: + sys.stdout.write("\nreject: ") + for a in args: + print(a) + #traceback.print_stack(None, 2) + solutionIndex = state["SolutionIndex"] if (state != None and "SolutionIndex" in state) else -1 + if solutionIndex != -1: + # If we have valid solutionIndex, this means we are during TensileCreateLibrary stage + # In this stage, all solutions in the logic should be valid + # So if any rejection happens, print the warning for further check + # This will be done only when --global-parameters=PrintSolutionRejectionReason=True + solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None + # if we don't have SolutionNameMin, we simply use the problemTypeName + solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin + raise Exception("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ + SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) + if state != None: + state["Valid"] = False + return False + +def matrixInstructionToMIParameters( + mi: list, + isa: IsaVersion, + wavefrontSize: int, + problemType: dict, + workGroup: list, + isaInfoMap: Dict[str, IsaInfo] + ): + """ + Converts a 9-item matrix instruction into the associated 4-item representation and + populates supporting MI parameters. + + Args: + mi: The matrix instruction to convert. Must have length 9. + isa: The ISA tuple. + wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. + problemType: The problem type dictionary. Typically "ProblemType" in a solution. + enableF32x: Whether to enable F32x. Typically "EnableF32XdlMathOp" in a solution. + """ + print1(f">> Converting MatrixInstruction {mi} to MI parameter:") + + if len(mi) != 9: + raise ValueError(f"MatrixInstruction must be 9 items long to convert into MI" + f" Parameters, found {mi} with length {len(mi)}") + + result = {} + + # Enable F32 XDL math operation only when the input type is f32. + enableF32xdl = ( + "F32XdlMathOp" in problemType + and not problemType["F32XdlMathOp"].isSingle() + and problemType["DataType"].isSingle() + ) + result["EnableF32XdlMathOp"] = enableF32xdl + + mi4 = [mi[0], mi[1], mi[2], mi[3]] + result["MatrixInstruction"] = mi4 + result["EnableMatrixInstruction"] = True + + waves = mi[7]* mi[8] + wg0 = mi[4] * mi[0] * mi[7] + + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] + result["ThreadTile"] = [1, 1] # dummy + + isSparse = problemType.get("Sparse", 0) + miDataType = DataType( + problemType["DataType"] + if not enableF32xdl + else problemType["F32XdlMathOp"] + ) + + result["MFMA_BF16_1K"] = ( + not isSparse + and isaInfoMap[isa].asmCaps["HasMFMA"] + and not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]) + and miDataType.isBFloat16() + and mi4 in validMFMA["B1k"] + ) + + # set MIBlock + MIBlockBM = wg0 // mi[0] + MIBlockBM = min(MIBlockBM, mi[3]) + MIBlockBN = mi[3] // MIBlockBM + result["MatrixInstBM"] = MIBlockBM + result["MatrixInstBN"] = MIBlockBN + result["MIBlock"] = [mi[0], mi[1], mi[2], mi[3], MIBlockBM, MIBlockBN] + + # set MIWaveGroup + miwg0 = min((wg0 // mi[0]) // MIBlockBM, waves) + result['MIWaveGroup'] = [miwg0, waves // miwg0] + + # set MIWaveTile + result['MIWaveTile'] = [mi[5], mi[6]] + + # set MIInputPerThread + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + result['MIInputPerThread'] = mi[0] * mi[2] * mi[3] // wavefrontSize + if (not hasMFMA) and hasWMMA and (isa[0] == 10 or isa[0] == 11): + result['MIInputPerThread'] = mi[2] + + sparseA = False if not isSparse else False if isSparse == 2 else True + sparseB = False if not isSparse else True if isSparse == 2 else False + result['MIInputPerThreadA'] = result['MIInputPerThread'] if not sparseA else result['MIInputPerThread'] // 2 + result['MIInputPerThreadB'] = result['MIInputPerThread'] if not sparseB else result['MIInputPerThread'] // 2 + result['MIInputPerThreadMetadata'] = result['MIInputPerThread'] if not isSparse else result['MIInputPerThread'] // 8 + result['Sparse'] = isSparse + + print1(f">> MI Parameters: {pprint.pformat(result)}") + return result diff --git a/tensilelite/Tensile/SolutionStructs/__init__.py b/tensilelite/Tensile/SolutionStructs/__init__.py new file mode 100644 index 0000000000..8c116b21f9 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/__init__.py @@ -0,0 +1,3 @@ +from .Solution import * +from .Utilities import * +from .Problem import * diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 48cbd509a6..8ecae196cd 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -99,9 +99,9 @@ def executeStepsInConfig( cCompiler, outputPath, buildTmpPath, - config["ShortNames"], + config["ShortNames"], debugConfig, - depthUConfig, + depthUConfig, deviceId, isaInfoMap, ) @@ -122,14 +122,14 @@ def executeStepsInConfig( else: libraryLogicConfig = {} LibraryLogic.main( - libraryLogicConfig, - srcToolchain.compiler, - outputPath, - debugConfig.splitGSU, + libraryLogicConfig, + srcToolchain.compiler, + outputPath, + debugConfig.splitGSU, debugConfig.printSolutionRejectionReason, debugConfig.printIndexAssignmentInfo, depthUConfig, - isaInfoMap, + isaInfoMap, ) print1("") else: @@ -442,11 +442,11 @@ def Tensile(userArgs): cxxCompiler, cCompiler, _, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) currentIsa = detectGlobalCurrentISA(device_id) isaInfoMap = makeIsaInfoMap([currentIsa], cxxCompiler) - assignGlobalParameters(config.get("GlobalParameters", {}), isaInfoMap, cxxCompiler) + assignGlobalParameters(config.get("GlobalParameters", {}), isaInfoMap) asmToolchain = makeAssemblyToolchain( cxxCompiler, - offloadBundler, + offloadBundler, args.CodeObjectVersion, ) srcToolchain = makeSourceToolchain( diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index eba0c1bd76..8ecff0e3c0 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -448,7 +448,7 @@ def generateLogicDataAndSolutions(logicFiles, args, assembler: Assembler, isaInf splitGSU = False printSolutionRejectionReason = False printIndexAssignmentInfo = False - + fIter = zip( logicFiles, itertools.repeat(assembler), @@ -565,11 +565,11 @@ def run(): targetIsas = [gfxToIsa(a) for a in archs] isaInfoMap = makeIsaInfoMap(targetIsas, cxxCompiler) - assignGlobalParameters(arguments, isaInfoMap, cxxCompiler) + assignGlobalParameters(arguments, isaInfoMap) asmToolchain = makeAssemblyToolchain( cxxCompiler, - offloadBundler, + offloadBundler, arguments["CodeObjectVersion"], arguments["BuildIdKind"] ) @@ -632,10 +632,10 @@ def validLogicFile(p: Path): kernelSerialNaming = getSerialNaming(kernels) kernelMinNaming = getMinNaming(kernels) kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, + kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, - DebugConfig(), + DebugConfig(), ) copyStaticFiles(outputPath) diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index 031759085c..72eb5638f3 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -26,28 +26,39 @@ import functools from pathlib import Path from multiprocessing import Pool -from typing import List - -from Tensile.Common import globalParameters, assignGlobalParameters, ParallelMap2, print1, printWarning +from typing import List, Dict + +from Tensile.Common import ( + globalParameters, + assignGlobalParameters, + ParallelMap2, + print1, + printWarning, + makeIsaInfoMap, + SUPPORTED_ISA, + gfxToIsa, + IsaVersion, + IsaInfo, +) from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig -from Tensile.SolutionStructs import Solution +from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters from .ParseArguments import parseArguments from .ValidMatrixInstruction import validateMatrixInstruction from .ValidWorkGroup import validateWorkGroup -def getParams(cxxCompiler): +def getParams(isaInfoMap, cxxCompiler): gp = globalParameters - assignGlobalParameters({"PrintSolutionRejectionReason": True}, cxxCompiler) + assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap, cxxCompiler) return gp -def handleCustomKernel(sol: dict) -> dict: +def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict: if not isCustomKernelConfig(sol): - return None + return sol name = sol["CustomKernelName"] print1(f">> Custom kernel: {name}") @@ -57,24 +68,19 @@ def handleCustomKernel(sol: dict) -> dict: mi = sol["MatrixInstruction"] if len(mi) != 9: - printWarning( - f"Custom kernel {name} has {len(mi)} matrix instructions. Expected 9." - ) + printWarning(f"Custom kernel {name} has MI length {len(mi)}, expected 9.") isa = sol["ISA"] wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] - enableF32x = sol.get("EnableF32XdlMathOp", False) - + workgroup = sol["WorkGroup"] - miParams = Solution.matrixInstructionToMIParameters( - mi, isa, wavefrontSize, ptype, enableF32x - ) + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) sol.update(miParams) return sol -def runChecks(logicPath: str, gp: dict, files: List[Path]): +def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): """ Run checks on the given files. @@ -96,12 +102,14 @@ def runChecks(logicPath: str, gp: dict, files: List[Path]): print1(f">> {file.relative_to(logicPath)}") for s in solutions: - s = handleCustomKernel(s) - if s: - keep += validateMatrixInstruction(s, gp, file.relative_to(logicPath)) - keep += validateWorkGroup(s, gp, file.relative_to(logicPath)) - else: - print1(f">> Skipping non-custom kernel...") + s = handleCustomKernel(s, isaInfoMap) + if all( + [ + validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), + ] + ): + keep += 1 total += 1 return keep, total @@ -115,8 +123,8 @@ def main(): jobs = int(args.Jobs) cxxCompiler = validateToolchain(args.CxxCompiler) - gp = globalParameters - assignGlobalParameters({"PrintSolutionRejectionReason": True}, cxxCompiler) + isaInfoMap = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) + assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) logicPath = Path(args.LogicPath) pattern = "**/*.yaml" @@ -125,19 +133,19 @@ def main(): batchSize = len(files) // jobs batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) - fn = functools.partial(runChecks, logicPath, gp) + fn = functools.partial(runChecks, logicPath, isaInfoMap) + keep, total = 0, 0 with Pool(processes=jobs) as pool: - results = pool.map(fn, batches) + results = pool.map_async(fn, batches) - # TIP: This is how to use joblib. Leave for reference. - # for _keep, _total in ParallelMap2( - # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" - # ): + # TIP: This is how to use joblib. Leave for reference. + # for _keep, _total in ParallelMap2( + # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" + # ): - # keep, total = 0, 0 - for _keep, _total in results: - keep += _keep - total += _total + for _keep, _total in results.get(): + keep += _keep + total += _total rejects = total - keep print(f"Total {total} solutions") diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index 3b24578e1e..ae18672c96 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -41,9 +41,9 @@ WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 Putting it all together: - - [32, 32, 1, 2, 1, 4, 1, 2, 2] - ^^^^^^^^^^^^ ^ ^^^^ ^^^^ - MatrixInst BlkM WT Wave + - [32, 32, 1, 2, 1, 4, 1, 2, 2] + ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + MatrixInst BlkM WT Wave - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used @@ -57,11 +57,12 @@ from inspect import currentframe, getframeinfo from Tensile.SolutionStructs import reject +from Tensile.Common import IsaVersion from Tensile.TensileInstructions.DataType import DataType from .Utilities import elineno -from Tensile.Common import IsaInfo +from Tensile.Common import IsaInfo, print1 MI_KEY: str = "MatrixInstruction" MI_ENABLED_KEY: str = "EnableMatrixInstruction" @@ -173,7 +174,9 @@ def elineno(): return f"{Path(frame.filename).name}:{frame.lineno}" -def validateMatrixInstruction(solution: dict, filepath: Path, isaInfoMap: Dict[str, IsaInfo]) -> bool: +def validateMatrixInstruction( + solution: dict, isaInfoMap: Dict[str, IsaInfo], filepath: Path +) -> bool: """ Validates the matrix instruction configured in the given solution. @@ -208,10 +211,14 @@ def validateMatrixInstruction(solution: dict, filepath: Path, isaInfoMap: Dict[s return False -def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): +def validateMIParameters( + solution: dict, isaInfoMap: Dict[str, IsaInfo], printSolutionRejectionReason: bool = True +): assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" - assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), elineno() + ": MI empty but enabled" + assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), ( + elineno() + ": MI empty but enabled" + ) isa = tuple(solution["ISA"]) @@ -220,6 +227,12 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): isa = (9, 4, 2) mi4 = solution[MI_KEY] + miEnabled = solution[MI_ENABLED_KEY] + assert len(mi4) == 4 or len(mi4) == 0, elineno() + ": MI length not 4 or 0" + if len(mi4) == 0: + assert miEnabled == False, elineno() + return + mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" mi9.append(solution["MatrixInstBM"]) @@ -228,14 +241,7 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): assert "MIWaveGroup" in solution, elineno() + ": missing MIWaveGroup" mi9.extend(solution["MIWaveGroup"]) - miEnabled = solution[MI_ENABLED_KEY] - - if len(mi4) == 0: - assert miEnabled == False, elineno() - else: - assert len(mi4) == 4 and len(mi9) == 9, ( - elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) - ) + assert len(mi4) == 4 and len(mi9) == 9, elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) if not miEnabled: return @@ -244,8 +250,7 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): wfsize = solution["WavefrontSize"] waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] - miwg0 = mi9[4] * mi9[0] * mi9[7] # Matrix instruction work group 0 - miwg1 = waves * wfsize // miwg0 + wg0 = mi9[4] * mi9[0] * mi9[7] # Work group 0 hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] @@ -263,18 +268,31 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): miWaveTile = solution["MIWaveTile"] # Check datatype - if not isSparse: - if hasMFMA: - if not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]): - if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: + if not isSparse: # If it's sparse + if hasMFMA: # and it supports MFMA + if not ( + miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()] + ): # but is invalid MFMA + print1( + f"Looks like {mi4} of type {miDataType.toChar()} is not supported for MFMA {validMFMA[miDataType.toChar()]}" + ) + if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA assert solution["MFMA_BF16_1K"], elineno() else: - reject(solution, f"Invalid MFMA BFloat16 configuration: {solution}") + return reject( + solution, + printSolutionRejectionReason, + f"Invalid MFMA BFloat16 configuration: {solution}", + ) elif hasWMMA and (not mi4 in validWMMA): - reject(solution, f"Invalid WMMA configuration: {solution}") + return reject( + solution, printSolutionRejectionReason, f"Invalid WMMA configuration: {solution}" + ) else: if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): - reject(solution, f"Invalid SMFMA configuration: {solution}") + return reject( + solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" + ) if (not hasMFMA) and hasWMMA: if isa[0] == 10 or isa[0] == 11: @@ -285,11 +303,11 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): assert miBlock[1] == mi4[1], elineno() assert miBlock[2] == mi4[2], elineno() assert miBlock[3] == mi4[3], elineno() - assert miBlock[4] == min(miwg0 // mi4[0], mi4[3]), elineno() + assert miBlock[4] == min(wg0 // mi4[0], mi4[3]), elineno() assert miBlock[5] == mi4[3] // miBlock[4], elineno() # Check MIWaveGroup - assert miWaveGroup[0] == min((miwg0 // mi4[0]) // miBlock[4], waves), elineno() + assert miWaveGroup[0] == min((wg0 // mi4[0]) // miBlock[4], waves), elineno() assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() # Check MIWaveTile @@ -313,3 +331,4 @@ def validateMIParameters(solution: dict, isaInfoMap: Dict[str, IsaInfo]): # assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() # assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() # assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() + return True diff --git a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py index 7de289396d..4761584d6a 100644 --- a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py +++ b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py @@ -29,6 +29,10 @@ Example: ( wg0 x wg1 x LocalSplitU ) """ +from typing import Dict + +from Tensile.Common import IsaVersion, IsaInfo + from .Utilities import elineno validWorkGroups = [] @@ -41,9 +45,9 @@ validWorkGroups.append(workGroup) -def validateWorkGroup(solution: dict, globalParams: dict, filepath: str): +def validateWorkGroup(solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: str): try: - _validateWorkGroup(solution, globalParams) + _validateWorkGroup(solution, isaInfoMap) assert solution["Valid"], f"Solution was rejected: {elineno()}" return True except AssertionError as e: @@ -53,6 +57,6 @@ def validateWorkGroup(solution: dict, globalParams: dict, filepath: str): return False -def _validateWorkGroup(solution: dict, globalParams: dict): +def _validateWorkGroup(solution: dict, isaInfoMap: dict): assert "WorkGroup" in solution, elineno() assert solution["WorkGroup"] in validWorkGroups, elineno() From 972426d9d74867bc52143ee880c859ff3f1400aa Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Wed, 26 Feb 2025 20:27:20 +0000 Subject: [PATCH 081/105] test: custom kernel config 9-4 conversion --- tensilelite/Tensile/BenchmarkProblems.py | 1 - .../Tensile/SolutionStructs/Utilities.py | 24 +- .../TensileLogic/ValidMatrixInstruction.py | 41 ++-- .../Tests/unit/testCustomKernelConfig.py | 205 ++++++++++++++++++ 4 files changed, 246 insertions(+), 25 deletions(-) create mode 100644 tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index be8013f9ba..5ec467fd6d 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -69,7 +69,6 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse solution = {} solution.update({ "ProblemType": deepcopy(problemType.state), - "ISA": isa, }) solution.update(constantParams) solution.update(perm) diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index f88597ead5..70b7b149f2 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -30,9 +30,20 @@ from Tensile.Common.ValidParameters import validMFMA from Tensile.TensileInstructions import DataType -def reject(state, printSolutionRejectionReason: bool = True, *args): +def reject(state: dict, printSolutionRejectionReason: bool = True, *args) -> bool: + """ + Reject a solution based on its internal state. + + Args: + state: The state of the solution. + printSolutionRejectionReason: If True, print the rejection reason. + *args: Additional arguments to print if rejection occurs. + + Returns: + True if the solution is rejected, False otherwise. + """ if state and "NoReject" in state and state["NoReject"]: - return + return False if printSolutionRejectionReason: sys.stdout.write("\nreject: ") for a in args: @@ -51,7 +62,7 @@ def reject(state, printSolutionRejectionReason: bool = True, *args): SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) if state != None: state["Valid"] = False - return False + return True def matrixInstructionToMIParameters( mi: list, @@ -59,7 +70,7 @@ def matrixInstructionToMIParameters( wavefrontSize: int, problemType: dict, workGroup: list, - isaInfoMap: Dict[str, IsaInfo] + isaInfoMap: Dict[IsaVersion, IsaInfo] ): """ Converts a 9-item matrix instruction into the associated 4-item representation and @@ -70,7 +81,6 @@ def matrixInstructionToMIParameters( isa: The ISA tuple. wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. problemType: The problem type dictionary. Typically "ProblemType" in a solution. - enableF32x: Whether to enable F32x. Typically "EnableF32XdlMathOp" in a solution. """ print1(f">> Converting MatrixInstruction {mi} to MI parameter:") @@ -79,6 +89,8 @@ def matrixInstructionToMIParameters( f" Parameters, found {mi} with length {len(mi)}") result = {} + result["ISA"] = isa + result["WavefrontSize"] = wavefrontSize # Enable F32 XDL math operation only when the input type is f32. enableF32xdl = ( @@ -95,6 +107,8 @@ def matrixInstructionToMIParameters( waves = mi[7]* mi[8] wg0 = mi[4] * mi[0] * mi[7] + print(f"### waves: {waves} wg0: {wg0} mi4: {mi4} mi: {mi}") + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] result["ThreadTile"] = [1, 1] # dummy diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index ae18672c96..fe3a0c9cfa 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -57,12 +57,12 @@ from inspect import currentframe, getframeinfo from Tensile.SolutionStructs import reject -from Tensile.Common import IsaVersion +from Tensile.Common import IsaVersion, IsaInfo, print1 +from Tensile.Common.Architectures import SUPPORTED_ISA from Tensile.TensileInstructions.DataType import DataType from .Utilities import elineno -from Tensile.Common import IsaInfo, print1 MI_KEY: str = "MatrixInstruction" MI_ENABLED_KEY: str = "EnableMatrixInstruction" @@ -212,7 +212,7 @@ def validateMatrixInstruction( def validateMIParameters( - solution: dict, isaInfoMap: Dict[str, IsaInfo], printSolutionRejectionReason: bool = True + solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], printSolutionRejectionReason: bool = True ): assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" @@ -220,12 +220,20 @@ def validateMIParameters( elineno() + ": MI empty but enabled" ) - isa = tuple(solution["ISA"]) - + isa = IsaVersion(*solution["ISA"]) + assert isa in SUPPORTED_ISA, elineno() + ": Unsupported ISA: " + str(isa) # TODO: Temporary until all 940/941 ISAs are removed if (9, 4, 0) <= isa <= (9, 4, 1): isa = (9, 4, 2) + ptype = solution["ProblemType"] + isSparse = ptype.get("Sparse", 0) + miDataType = DataType( + ptype["DataType"] + if not solution.get("EnableF32XdlMathOp", False) + else ptype["F32XdlMathOp"] + ) + mi4 = solution[MI_KEY] miEnabled = solution[MI_ENABLED_KEY] assert len(mi4) == 4 or len(mi4) == 0, elineno() + ": MI length not 4 or 0" @@ -233,6 +241,8 @@ def validateMIParameters( assert miEnabled == False, elineno() return + assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" + mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" mi9.append(solution["MatrixInstBM"]) @@ -244,9 +254,8 @@ def validateMIParameters( assert len(mi4) == 4 and len(mi9) == 9, elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) if not miEnabled: - return + return False - assert mi4 in validMatrixInstructions, elineno() wfsize = solution["WavefrontSize"] waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] @@ -255,13 +264,6 @@ def validateMIParameters( hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] - ptype = solution["ProblemType"] - isSparse = ptype.get("Sparse", 0) - miDataType = DataType( - ptype["DataType"] - if not solution.get("EnableF32XdlMathOp", False) - else ptype["F32XdlMathOp"] - ) miBlock = solution["MIBlock"] miWaveGroup = solution["MIWaveGroup"] @@ -279,18 +281,18 @@ def validateMIParameters( if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA assert solution["MFMA_BF16_1K"], elineno() else: - return reject( + return not reject( solution, printSolutionRejectionReason, f"Invalid MFMA BFloat16 configuration: {solution}", ) elif hasWMMA and (not mi4 in validWMMA): - return reject( + return not reject( solution, printSolutionRejectionReason, f"Invalid WMMA configuration: {solution}" ) else: if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): - return reject( + return not reject( solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" ) @@ -318,10 +320,11 @@ def validateMIParameters( miInputPerThread = solution["MIInputPerThread"] # If Navi architecture, the input per thread is different - if (10, 0, 0) <= isa <= (11, 0, 2): + if IsaVersion(10, 0, 0) <= isa <= IsaVersion(11, 0, 2): assert miInputPerThread == mi4[2], elineno() else: - assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, elineno() + assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, f"{elineno()} MIInputPerThread: {miInputPerThread} != {mi4[0]} * {mi4[2]} * {mi4[3]} / {wfsize} = {mi4[0] * mi4[2] * mi4[3] // wfsize}" + # miInputPerThreadA = solution["MIInputPerThreadA"] # miInputPerThreadB = solution["MIInputPerThreadB"] diff --git a/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py new file mode 100644 index 0000000000..90cbe52d28 --- /dev/null +++ b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py @@ -0,0 +1,205 @@ +import pytest +import yaml +from pprint import pformat + +from Tensile.CustomKernels import getCustomKernelConfig +from Tensile.SolutionStructs import matrixInstructionToMIParameters +from Tensile.Common.Architectures import SUPPORTED_ISA +from Tensile.Common.Capabilities import makeIsaInfoMap +from Tensile.Common.Types import IsaVersion +from Tensile.Common.GlobalParameters import defaultSolution +from Tensile.Toolchain.Validators import validateToolchain +from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters + +cxxCompiler = validateToolchain("amdclang++") + +ISA_INFO_MAP = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) + + +# @pytest.mark.parametrize("objs", [("TestKernel", testKernelDir, configResult)]) +def testConvert9ItemCustomKernelConfig(): + + inputConf = yaml.load( + """ +custom.config: + ProblemType: + OperationType: GEMM + DataTypeA: f8n + DataTypeB: h + UseScaleAB: "Scalar" + DataType: h + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: False + UseBias: 1 + Activation: True + UseScaleAlphaVec: 1 + UseBeta: True + Batched: True + GroupedGemm: True + SupportUserArgs: True + MatrixInstruction: [32, 32, 8, 1, 5, 6, 7, 8, 9] + 1LDSBuffer: 1 + ScheduleIterAlg: 3 + DepthU: 32 + StaggerU: 0 + WorkGroupMapping: 8 + WaveSeparateGlobalReadA: 1 + WaveSeparateGlobalReadB: 1 + GlobalReadVectorWidthA: 4 + GlobalReadVectorWidthB: 2 + AssertFree0ElementMultiple: 4 + AssertSummationElementMultiple: 1 + NoReject: 1 + InternalSupportParams: + KernArgsVersion: 0 + SupportUserGSU: False + SupportCustomWGM: False + SupportCustomStaggerU: False + UseUniversalArgs: False +""", + yaml.SafeLoader, + ) + inputConf = inputConf["custom.config"] + + isa = IsaVersion(9, 4, 2) + wavefrontSize = 48 + workGroup = [4, 5, 6] + + outputConf = matrixInstructionToMIParameters( + inputConf["MatrixInstruction"], + isa, + wavefrontSize, + inputConf["ProblemType"], + workGroup, + ISA_INFO_MAP, + ) + + input = { + "MatrixInstruction": inputConf["MatrixInstruction"], + } + + print("inputConf: ", pformat(input)) + print("outputConf: ", pformat(outputConf)) + + assert outputConf["MatrixInstruction"] == [32, 32, 8, 1] + assert outputConf["EnableMatrixInstruction"] == True + assert outputConf["MIBlock"] == [32, 32, 8, 1, 1, 1] + assert outputConf["MIWaveGroup"] == [40, 1] + assert outputConf["MIWaveTile"] == [6, 7] + assert outputConf["MatrixInstBM"] == 1 + assert outputConf["MatrixInstBN"] == 1 + assert outputConf["MIInputPerThread"] == 5 + assert outputConf["MIInputPerThreadA"] == 5 + assert outputConf["MIInputPerThreadB"] == 5 + assert outputConf["MIInputPerThreadMetadata"] == 5 + assert outputConf["ThreadTile"] == [1, 1] + assert outputConf["Sparse"] == 0 + assert outputConf["WorkGroup"] == [1280, 2, 6] # Why do we change the workgroup here? + assert outputConf["WavefrontSize"] == 48 + assert outputConf["ISA"] == isa + assert outputConf["EnableF32XdlMathOp"] == False + assert outputConf["MFMA_BF16_1K"] == False + + solution = defaultSolution + solution.update(inputConf) + solution.update(outputConf) + + assert validateMIParameters(solution, ISA_INFO_MAP, True) == True + +""" +def testConvert4ItemCustomKernelConfig(): + + inputConf = yaml.load( + """ +custom.config: + ProblemType: + OperationType: GEMM + DataTypeA: f8n + DataTypeB: h + UseScaleAB: "Scalar" + DataType: h + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: False + UseBias: 1 + Activation: True + UseScaleAlphaVec: 1 + UseBeta: True + Batched: True + GroupedGemm: True + SupportUserArgs: True + MatrixInstruction: [32, 32, 8, 1, 5, 6, 7, 8, 9] + 1LDSBuffer: 1 + ScheduleIterAlg: 3 + DepthU: 32 + StaggerU: 0 + WorkGroupMapping: 8 + WaveSeparateGlobalReadA: 1 + WaveSeparateGlobalReadB: 1 + GlobalReadVectorWidthA: 4 + GlobalReadVectorWidthB: 2 + AssertFree0ElementMultiple: 4 + AssertSummationElementMultiple: 1 + NoReject: 1 + InternalSupportParams: + KernArgsVersion: 0 + SupportUserGSU: False + SupportCustomWGM: False + SupportCustomStaggerU: False + UseUniversalArgs: False +""", + yaml.SafeLoader, + ) + inputConf = inputConf["custom.config"] + + isa = IsaVersion(9, 4, 2) + wavefrontSize = 48 + workGroup = [4, 5, 6] + + outputConf = matrixInstructionToMIParameters( + inputConf["MatrixInstruction"], + isa, + wavefrontSize, + inputConf["ProblemType"], + workGroup, + ISA_INFO_MAP, + ) + + input = { + "MatrixInstruction": inputConf["MatrixInstruction"], + } + + print("inputConf: ", pformat(input)) + print("outputConf: ", pformat(outputConf)) + + assert outputConf["MatrixInstruction"] == [32, 32, 8, 1] + assert outputConf["EnableMatrixInstruction"] == True + assert outputConf["MIBlock"] == [32, 32, 8, 1, 1, 1] + assert outputConf["MIWaveGroup"] == [40, 1] + assert outputConf["MIWaveTile"] == [6, 7] + assert outputConf["MatrixInstBM"] == 1 + assert outputConf["MatrixInstBN"] == 1 + assert outputConf["MIInputPerThread"] == 5 + assert outputConf["MIInputPerThreadA"] == 5 + assert outputConf["MIInputPerThreadB"] == 5 + assert outputConf["MIInputPerThreadMetadata"] == 5 + assert outputConf["ThreadTile"] == [1, 1] + assert outputConf["Sparse"] == 0 + assert outputConf["WorkGroup"] == [1280, 2, 6] # Why do we change the workgroup here? + assert outputConf["WavefrontSize"] == 48 + assert outputConf["ISA"] == isa + assert outputConf["EnableF32XdlMathOp"] == False + assert outputConf["MFMA_BF16_1K"] == False + + solution = defaultSolution + solution.update(inputConf) + solution.update(outputConf) + + assert validateMIParameters(solution, ISA_INFO_MAP, True) == True + +""" From 39fffa3cddb8896cb363df1e41e40f03d8f284f2 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 27 Feb 2025 00:07:59 +0000 Subject: [PATCH 082/105] draft: continue migrating custom kernel MI entries --- .../Tensile/TensileLogic/ParseArguments.py | 2 +- tensilelite/Tensile/TensileLogic/Run.py | 8 +++- .../Tests/unit/testCustomKernelConfig.py | 42 +------------------ 3 files changed, 8 insertions(+), 44 deletions(-) diff --git a/tensilelite/Tensile/TensileLogic/ParseArguments.py b/tensilelite/Tensile/TensileLogic/ParseArguments.py index 2b5af8ccc9..c9db912875 100644 --- a/tensilelite/Tensile/TensileLogic/ParseArguments.py +++ b/tensilelite/Tensile/TensileLogic/ParseArguments.py @@ -28,7 +28,7 @@ from Tensile.Toolchain.Validators import ToolchainDefaults -def parseArguments() -> Dict[str, Any]: +def parseArguments(): """ Returns: A dictionary containing the keys representing options and their values. diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index 72eb5638f3..8f023b054c 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -56,9 +56,9 @@ def getParams(isaInfoMap, cxxCompiler): return gp -def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict: +def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict | None: if not isCustomKernelConfig(sol): - return sol + return None name = sol["CustomKernelName"] print1(f">> Custom kernel: {name}") @@ -103,6 +103,10 @@ def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List for s in solutions: s = handleCustomKernel(s, isaInfoMap) + + if s is None: + continue + if all( [ validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), diff --git a/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py index 90cbe52d28..9924f4b9af 100644 --- a/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py +++ b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py @@ -112,47 +112,7 @@ def testConvert9ItemCustomKernelConfig(): """ def testConvert4ItemCustomKernelConfig(): - inputConf = yaml.load( - """ -custom.config: - ProblemType: - OperationType: GEMM - DataTypeA: f8n - DataTypeB: h - UseScaleAB: "Scalar" - DataType: h - DestDataType: s - ComputeDataType: s - HighPrecisionAccumulate: True - TransposeA: False - TransposeB: False - UseBias: 1 - Activation: True - UseScaleAlphaVec: 1 - UseBeta: True - Batched: True - GroupedGemm: True - SupportUserArgs: True - MatrixInstruction: [32, 32, 8, 1, 5, 6, 7, 8, 9] - 1LDSBuffer: 1 - ScheduleIterAlg: 3 - DepthU: 32 - StaggerU: 0 - WorkGroupMapping: 8 - WaveSeparateGlobalReadA: 1 - WaveSeparateGlobalReadB: 1 - GlobalReadVectorWidthA: 4 - GlobalReadVectorWidthB: 2 - AssertFree0ElementMultiple: 4 - AssertSummationElementMultiple: 1 - NoReject: 1 - InternalSupportParams: - KernArgsVersion: 0 - SupportUserGSU: False - SupportCustomWGM: False - SupportCustomStaggerU: False - UseUniversalArgs: False -""", + inputConf = yaml.load("1234", yaml.SafeLoader, ) inputConf = inputConf["custom.config"] From 70054c3b131f55b97c54c2e1008b595e9d099454 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 27 Feb 2025 21:17:50 +0000 Subject: [PATCH 083/105] draft: custom TensileLogic for custom kernel manipulation --- tensilelite/Tensile/BenchmarkProblems.py | 1 + .../Tensile/Common/GlobalParameters.py | 1 - tensilelite/Tensile/Common/ValidParameters.py | 24 +- .../Tensile/SolutionStructs/Utilities.py | 10 +- .../TensileLogicCUSTOM/ParseArguments.py | 62 ++++ tensilelite/Tensile/TensileLogicCUSTOM/Run.py | 250 +++++++++++++ .../Tensile/TensileLogicCUSTOM/Utilities.py | 34 ++ .../ValidMatrixInstruction.py | 342 ++++++++++++++++++ .../TensileLogicCUSTOM/ValidWorkGroup.py | 62 ++++ .../Tensile/TensileLogicCUSTOM/__init__.py | 1 + tensilelite/Tensile/bin/TensileLogic | 6 +- 11 files changed, 784 insertions(+), 9 deletions(-) create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/Run.py create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py create mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/__init__.py diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 5ec467fd6d..4c4a1bfeb7 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -122,6 +122,7 @@ def _getCustomKernelSolutionObj( ptype = sol["ProblemType"] workgroup = sol["WorkGroup"] + # TODO: this should be deleted once all custom kernel configs MI are length 4. miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) sol.update(miParams) diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index ba27cd4dd1..0ebf9a4227 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -502,7 +502,6 @@ def capRow(isaInfoMap, cap, capType): for info in isaInfoMap.values() ] - print1(f"infomap: {isaInfoMap}") gfxs = list(map(isaToGfx, isaInfoMap.keys())) headerRow = ["Capability"] + gfxs diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index fad16def34..cd43cc10ea 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -748,7 +748,7 @@ # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm "KernelLanguage": ["Assembly"], # We set validParams["ISA"] in multiple places - "ISA": validISA, # arch for assembly kernels + "ISA": validISA, # arch for assembly kernels # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. @@ -782,6 +782,28 @@ "ConvertAfterDS": [False, True], # Force disable shadow init to release more sgpr in preloop "ForceDisableShadowInit": [False, True], + "EnableF32XdlMathOp": [False, True], + 'EnableMatrixInstruction': [False, True], + 'ISA': -1, + 'MFMA_BF16_1K': [False, True], + 'MIBlock': -1, + 'MIInputPerThread': -1, + 'MIInputPerThreadA': -1, + 'MIInputPerThreadB': -1, + 'MIInputPerThreadMetadata': -1, + 'MIWaveGroup': -1, + 'MIWaveTile': -1, + 'MatrixInstM': -1, + 'MatrixInstN': -1, + 'MatrixInstK': -1, + 'MatrixInstB': -1, + 'MatrixInstBM': -1, + 'MatrixInstBN': -1, + 'MatrixInstruction': -1, + 'Sparse': -1, + 'ThreadTile': -1, + 'WavefrontSize': -1, + 'WorkGroup': -1, } diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index 70b7b149f2..4f0be9e675 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -26,7 +26,7 @@ import pprint from typing import Dict -from Tensile.Common import IsaVersion, IsaInfo, print1 +from Tensile.Common import IsaVersion, IsaInfo, print1, print2 from Tensile.Common.ValidParameters import validMFMA from Tensile.TensileInstructions import DataType @@ -82,7 +82,7 @@ def matrixInstructionToMIParameters( wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. problemType: The problem type dictionary. Typically "ProblemType" in a solution. """ - print1(f">> Converting MatrixInstruction {mi} to MI parameter:") + print1(f">> --DBG-- Converting MatrixInstruction {mi} to MI parameters") if len(mi) != 9: raise ValueError(f"MatrixInstruction must be 9 items long to convert into MI" @@ -103,12 +103,14 @@ def matrixInstructionToMIParameters( mi4 = [mi[0], mi[1], mi[2], mi[3]] result["MatrixInstruction"] = mi4 result["EnableMatrixInstruction"] = True + result["MatrixInstM"] = mi[0] + result["MatrixInstN"] = mi[1] + result["MatrixInstK"] = mi[2] + result["MatrixInstB"] = mi[3] waves = mi[7]* mi[8] wg0 = mi[4] * mi[0] * mi[7] - print(f"### waves: {waves} wg0: {wg0} mi4: {mi4} mi: {mi}") - result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] result["ThreadTile"] = [1, 1] # dummy diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py b/tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py new file mode 100644 index 0000000000..015485b2c8 --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py @@ -0,0 +1,62 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from argparse import ArgumentParser +from typing import Any, Dict + +from Tensile.Toolchain.Validators import ToolchainDefaults + + +def parseArguments(): + """ + Returns: + A dictionary containing the keys representing options and their values. + """ + + argParser = ArgumentParser( + description="TensileValidateLogic runs critical checks to ensure the " + "integrity of the supplied logic files.", + ) + + argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") + argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") + argParser.add_argument("-v", "--verbose", dest="Verbose", type=int, default=1, choices=[0, 1, 2, 3], help="Set print level with ``--v 2``.") + argParser.add_argument( + "--jobs", + "-j", + dest="Jobs", + action="store", + default=48, + help="Number of worker processes to use during validation checks.", + ) + argParser.add_argument( + "--cxx-compiler", + dest="CxxCompiler", + action="store", + default=ToolchainDefaults.CXX_COMPILER, + help=f"Default: {ToolchainDefaults.CXX_COMPILER}", + ) + args = argParser.parse_args() + + return args diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/Run.py b/tensilelite/Tensile/TensileLogicCUSTOM/Run.py new file mode 100644 index 0000000000..84abbb0942 --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/Run.py @@ -0,0 +1,250 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + + +import yaml +import functools +from pathlib import Path +from multiprocessing import Pool +from typing import List, Dict + +from Tensile.Common import ( + globalParameters, + assignGlobalParameters, + ParallelMap2, + print1, + print2, + printWarning, + makeIsaInfoMap, + SUPPORTED_ISA, + gfxToIsa, + IsaVersion, + IsaInfo, + verbosity +) + +from Tensile.LibraryIO import readYAML +from Tensile.Toolchain.Validators import validateToolchain +from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig +from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters +from Tensile import CUSTOM_KERNEL_PATH + +from .ParseArguments import parseArguments +from .ValidMatrixInstruction import validateMatrixInstruction +from .ValidWorkGroup import validateWorkGroup + + +def getParams(isaInfoMap, cxxCompiler): + gp = globalParameters + assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap, cxxCompiler) + return gp + + +def handleCustomKernel(sol: dict, isaInfoMap: dict): + if not isCustomKernelConfig(sol): + return None + + name = sol["CustomKernelName"] + + dir = CUSTOM_KERNEL_PATH + # dir = str(Path(CUSTOM_KERNEL_PATH)/".."/"NEWCustomKernels") + custom = getCustomKernelConfig(name, {}, dir) + sol.update(custom) + + mi = sol["MatrixInstruction"] + print1(f">> FOUND Custom kernel: {name} with MI {mi}") + + if len(mi) == 4: + print1(f">> --DBG-- -> Success, no need to convert, has MI length {len(mi)}\n---") + return None + + isa = IsaVersion(*sol["ISA"]) + wavefrontSize = sol["WavefrontSize"] + ptype = sol["ProblemType"] + workgroup = sol["WorkGroup"] + + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + + ## Experimental custom kernel config checker code + import pprint + yamlstring = convert_pformat_to_condensed(pprint.pformat(miParams)) + print(yamlstring) + + ## end of experimetnal section + + # try: + # miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + # except Exception as e: + # printWarning(f"Custom kernel {name} failed to convert MI to parameters: {e}") + # return None + sol.update(miParams) + return sol, yamlstring + + +def convert_pformat_to_condensed(pformat_str): + # Remove the dictionary braces and split into lines + lines = pformat_str.strip('{}').split('\n') + + # Initialize an empty list to store the formatted lines + formatted_lines = [] + + # Iterate over each line + for line in lines: + # Remove leading and trailing whitespace + line = line.strip() + + # Remove the 'ISA' line as it is not needed in the output + if line.startswith("'ISA'"): + continue + + line = line.replace("'", "") + line = line[:-1] if line.endswith(',') else line + + + line = ' ' + line + + # Append the formatted line to the list + formatted_lines.append(line) + + # Join the formatted lines into a single string + result = '\n'.join(formatted_lines) + + return result + +def replace_line_in_file(file_path, search_string, replacement_string): + """ + Replaces a line in a file that matches the search_string with the replacement_string. + + Args: + file_path (str): The path to the file. + search_string (str): The string to search for in the file. + replacement_string (str): The multi-line string to replace the matching line with. + """ + # Read the file contents + with open(file_path, 'r') as file: + lines = file.readlines() + + # Open the file in write mode to overwrite the contents + with open(file_path, 'w') as file: + for line in lines: + if search_string in line: + # Replace the matching line with the replacement string + file.write(replacement_string + '\n') + else: + # Write the original line + file.write(line) + +def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): + """ + Run checks on the given files. + + Args: + logicPath: Path to the logic directory. + gp: Global parameters. + files: List of files to check. + + Returns: + Tuple of (keep, total) where keep is the number of solutions to keep and + total is the total number of solutions. + """ + keep, total = 0, 0 + for file in files: + if "Experimental" in file.parts: + return keep, total + + + solutions = readYAML(file)[5] # Solutions are the 5th index + + # print1(f">> {file.relative_to(logicPath)}") + for s in solutions: + s = handleCustomKernel(s, isaInfoMap) + + if s is None: + continue + + sol, replacement_string = s + customfile = Path(CUSTOM_KERNEL_PATH) / (sol["CustomKernelName"] + ".s") + print1(f"## UPDATING {customfile}") + replace_line_in_file(customfile, " MatrixInstruction:", replacement_string) + + + print1(f"## Custom kernel {file.relative_to(logicPath)}") + if all( + [ + validateMatrixInstruction(sol, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(sol, isaInfoMap, file.relative_to(logicPath)), + ] + ): + keep += 1 + total += 1 + return keep, total + + +def main(): + args = parseArguments() + if not any([args.Check]): + print1("No checks specified. Exiting.") + exit(0) + + global verbosity + verbosity = args.Verbose + + jobs = int(args.Jobs) + cxxCompiler = validateToolchain(args.CxxCompiler) + + isaInfoMap = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) + assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) + + logicPath = Path(args.LogicPath) + if logicPath.is_file() and logicPath.suffix == ".yaml": + files = [logicPath] + else: + pattern = "**/*.yaml" + files = list(logicPath.glob(pattern)) + + batchSize = len(files) // min(len(files), jobs) + batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) + + fn = functools.partial(runChecks, logicPath, isaInfoMap) + keep, total = 0, 0 + # with Pool(processes=jobs) as pool: + # results = pool.map_async(fn, batches) + + # # TIP: This is how to use joblib. Leave for reference. + results = ParallelMap2( + fn, batches, multiArg=False, procs=jobs, return_as="list" + ) + + for _keep, _total in results: + # for _keep, _total in results.get(): + keep += _keep + total += _total + + rejects = total - keep + print(f"Total {total} solutions") + print(f"Keep {keep} solutions") + print(f"Reject {rejects} solutions") + + if rejects > 0: + exit(1) diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py b/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py new file mode 100644 index 0000000000..4d91143379 --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py @@ -0,0 +1,34 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from pathlib import Path +from inspect import currentframe, getframeinfo + + +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py new file mode 100644 index 0000000000..0f0e4410ec --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py @@ -0,0 +1,342 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidMatrixInstruction +--- +Format: (M x N x K x B) + XDLOPS tile definition, only valid for gfx908, gfx90a + MxNxKxB specifies matrix instruction variants + MxNxB determines the shape of the C tile each instruction worked on + K determines the unroll depth + +Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + (Note: MxN means M-by-N in the following comments) + MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + Putting it all together: + - [32, 32, 1, 2, 1, 4, 1, 2, 2] + ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + MatrixInst BlkM WT Wave + - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + +Notes: + - If empty, do not use these instructions +""" + +import math +from typing import Dict +from pathlib import Path +from inspect import currentframe, getframeinfo + +from Tensile.SolutionStructs import reject +from Tensile.Common import IsaVersion, IsaInfo, print1 +from Tensile.Common.Architectures import SUPPORTED_ISA +from Tensile.TensileInstructions.DataType import DataType + +from .Utilities import elineno + + +MI_KEY: str = "MatrixInstruction" +MI_ENABLED_KEY: str = "EnableMatrixInstruction" + + +validMFMA = {} +validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] +validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] +validMFMA["4xi8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], + [32, 32, 16, 1], + [16, 16, 32, 1], +] +validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] +validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["C"] = validMFMA["S"] +validMFMA["Z"] = validMFMA["D"] +validMFMA["I8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], +] + [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] +validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8"] = validMFMA["F8"] +validMFMA["F8B8"] = validMFMA["F8"] +validMFMA["B8F8"] = validMFMA["F8"] +validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8N"] = validMFMA["F8N"] +validMFMA["F8B8N"] = validMFMA["F8N"] +validMFMA["B8F8N"] = validMFMA["F8N"] +validWMMA = [ + [16, 16, 16, 1], +] +validTT = 32 +validMFMA["_format9"] = [] + +for MFMA in [ + validMFMA["H"], + validMFMA["S"], + validMFMA["B"], + validMFMA["D"], + validMFMA["X"], + validMFMA["F8N"], + validWMMA, +]: + for MI in MFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validMatrixInstructions = ( + [[], [-1]] + + validMFMA["H"] + + validMFMA["S"] + + validMFMA["B"] + + validMFMA["D"] + + validMFMA["B1k"] + + validMFMA["X"] +) +validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] + +validSMFMA = {} +validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["I8"] = validSMFMA["4xi8"] +validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8"] = validSMFMA["F8"] +validSMFMA["F8B8"] = validSMFMA["F8"] +validSMFMA["B8F8"] = validSMFMA["F8"] +validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8N"] = validSMFMA["F8N"] +validSMFMA["F8B8N"] = validSMFMA["F8N"] +validSMFMA["B8F8N"] = validSMFMA["F8N"] +validSMFMA["_format9"] = [] +for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: + for MI in SMFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validSMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] +validMatrixInstructions = ( + validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] +) + + +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" + + +def validateMatrixInstruction( + solution: dict, isaInfoMap: Dict[str, IsaInfo], filepath: Path +) -> bool: + """ + Validates the matrix instruction configured in the given solution. + + The function performs the following checks: + - Ensures that the solution contains the required keys for matrix instruction support. + - Ensures that the matrix instruction is not empty when it is enabled. + - Validates that the matrix instruction is in the list of valid matrix instructions. + - If the matrix instruction has 9 elements, it performs detailed validation checks: + - Validates the work group dimensions. + - Checks if the matrix instruction is supported by the assembler capabilities (MFMA or WMMA). + - Validates the input per thread for sparse and non-sparse configurations. + - Validates the matrix instruction block, wave group, and wave tile dimensions. + - If the matrix instruction has 4 elements, it ensures that matrix instructions are enabled. + - If the matrix instruction is empty, it ensures that matrix instructions are disabled. + + Args: + solution: The solution to validate. + filepath: The path to the file containing the solution. + params: The global parameters for the solution. + + Raises: + AssertionError: If any of the validation checks fail. + """ + try: + validateMIParameters(solution, isaInfoMap) + assert solution["Valid"], f"Solution was rejected: {elineno()}" + return True + except AssertionError as e: + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) + return False + + +def validateMIParameters( + solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], printSolutionRejectionReason: bool = True +): + assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" + assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" + assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), ( + elineno() + ": MI empty but enabled" + ) + + isa = IsaVersion(*solution["ISA"]) + assert isa in SUPPORTED_ISA, elineno() + ": Unsupported ISA: " + str(isa) + # TODO: Temporary until all 940/941 ISAs are removed + if (9, 4, 0) <= isa <= (9, 4, 1): + isa = (9, 4, 2) + + ptype = solution["ProblemType"] + isSparse = ptype.get("Sparse", 0) + miDataType = DataType( + ptype["DataType"] + if not solution.get("EnableF32XdlMathOp", False) + else ptype["F32XdlMathOp"] + ) + + mi4 = solution[MI_KEY] + miEnabled = solution[MI_ENABLED_KEY] + assert len(mi4) == 4 or len(mi4) == 0, elineno() + ": MI length not 4 or 0" + if len(mi4) == 0: + assert miEnabled == False, elineno() + return + + assert solution["MatrixInstM"] == mi4[0] + assert solution["MatrixInstN"] == mi4[1] + assert solution["MatrixInstK"] == mi4[2] + assert solution["MatrixInstB"] == mi4[3] + + assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" + + mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] + assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" + mi9.append(solution["MatrixInstBM"]) + assert "MIWaveTile" in solution, elineno() + ": missing MIWaveTile" + mi9.extend(solution["MIWaveTile"]) + assert "MIWaveGroup" in solution, elineno() + ": missing MIWaveGroup" + mi9.extend(solution["MIWaveGroup"]) + + assert len(mi4) == 4 and len(mi9) == 9, elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) + + if not miEnabled: + return False + + + wfsize = solution["WavefrontSize"] + waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] + wg0 = mi9[4] * mi9[0] * mi9[7] # Work group 0 + + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + + miBlock = solution["MIBlock"] + miWaveGroup = solution["MIWaveGroup"] + miWaveTile = solution["MIWaveTile"] + + # Check datatype + if not isSparse: # If it's sparse + if hasMFMA: # and it supports MFMA + if not ( + miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()] + ): # but is invalid MFMA + print1( + f"Looks like {mi4} of type {miDataType.toChar()} is not supported for MFMA {validMFMA[miDataType.toChar()]}" + ) + if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA + assert solution["MFMA_BF16_1K"], elineno() + else: + return not reject( + solution, + True,#printSolutionRejectionReason, + f"Invalid MFMA BFloat16 configuration: {solution}", + ) + elif hasWMMA and (not mi4 in validWMMA): + return not reject( + solution, printSolutionRejectionReason, f"Invalid WMMA configuration: {solution}" + ) + else: + if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): + return not reject( + solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" + ) + + if (not hasMFMA) and hasWMMA: + if isa[0] == 10 or isa[0] == 11: + assert miInputPerThread == mi4[2], elineno() + + # Check MIBlock + assert miBlock[0] == mi4[0], elineno() + assert miBlock[1] == mi4[1], elineno() + assert miBlock[2] == mi4[2], elineno() + assert miBlock[3] == mi4[3], elineno() + assert miBlock[4] == min(wg0 // mi4[0], mi4[3]), elineno() + assert miBlock[5] == mi4[3] // miBlock[4], elineno() + + # Check MIWaveGroup + assert miWaveGroup[0] == min((wg0 // mi4[0]) // miBlock[4], waves), elineno() + assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() + + # Check MIWaveTile + assert miWaveTile[0] == mi9[5], elineno() + assert miWaveTile[1] == mi9[6], elineno() + + # Check MIInputPerThread + miInputPerThread = solution["MIInputPerThread"] + + # If Navi architecture, the input per thread is different + if IsaVersion(10, 0, 0) <= isa <= IsaVersion(11, 0, 2): + assert miInputPerThread == mi4[2], elineno() + else: + assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, f"{elineno()} MIInputPerThread: {miInputPerThread} != {mi4[0]} * {mi4[2]} * {mi4[3]} / {wfsize} = {mi4[0] * mi4[2] * mi4[3] // wfsize}" + + + # miInputPerThreadA = solution["MIInputPerThreadA"] + # miInputPerThreadB = solution["MIInputPerThreadB"] + # miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] + # sparseA = not isSparse if isSparse != 2 else False + # sparseB = isSparse == 2 if isSparse else False + # assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() + # assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() + # assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() + return True diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py new file mode 100644 index 0000000000..4761584d6a --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py @@ -0,0 +1,62 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidWorkGroup +--- +Dimensions of the workgroup which will operate on a tile and share lds +Example: ( wg0 x wg1 x LocalSplitU ) +""" + +from typing import Dict + +from Tensile.Common import IsaVersion, IsaInfo + +from .Utilities import elineno + +validWorkGroups = [] +for numThreads in range(32, 1025, 32): + for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: + for sg0 in range(1, numThreads // nsg + 1): + sg1 = numThreads // nsg // sg0 + if sg0 * sg1 * nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + + +def validateWorkGroup(solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: str): + try: + _validateWorkGroup(solution, isaInfoMap) + assert solution["Valid"], f"Solution was rejected: {elineno()}" + return True + except AssertionError as e: + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) + return False + + +def _validateWorkGroup(solution: dict, isaInfoMap: dict): + assert "WorkGroup" in solution, elineno() + assert solution["WorkGroup"] in validWorkGroups, elineno() diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/__init__.py b/tensilelite/Tensile/TensileLogicCUSTOM/__init__.py new file mode 100644 index 0000000000..46b158ded5 --- /dev/null +++ b/tensilelite/Tensile/TensileLogicCUSTOM/__init__.py @@ -0,0 +1 @@ +from .Run import main diff --git a/tensilelite/Tensile/bin/TensileLogic b/tensilelite/Tensile/bin/TensileLogic index d1b70ed72c..77effcf2b3 100755 --- a/tensilelite/Tensile/bin/TensileLogic +++ b/tensilelite/Tensile/bin/TensileLogic @@ -27,7 +27,7 @@ # This script only gets called by CMake try: - from Tensile import TensileLogic + from Tensile import TensileLogicCUSTOM except ImportError: import os.path import sys @@ -35,10 +35,10 @@ except ImportError: print(parentdir) sys.path.append(parentdir) - from Tensile import TensileLogic + from Tensile import TensileLogicCUSTOM ################################################################################ # Main ################################################################################ if __name__ == "__main__": - TensileLogic.main() + TensileLogicCUSTOM.main() From a4e3dd5eca5930284f05cf6d799a04f2b0945482 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Thu, 27 Feb 2025 21:21:26 +0000 Subject: [PATCH 084/105] feat: use mi parameters in custom kernels --- ...T128x16x128_MI16x16x1_44_Freesize_gfx942.s | 22 +- ...HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s | 22 +- ...HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname10_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname11_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname12_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname13_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname14_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname15_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname16_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname17_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname18_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname19_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname20_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname10_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname11_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname12_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname13_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname14_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname15_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname16_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname17_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname18_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname19_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s | 22 +- ...G_AS_SAB_SAV_UserArgs_shortname20_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s | 22 +- ...GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname0_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname10_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname11_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname12_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname13_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname14_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname15_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname16_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname17_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname18_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname1_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname2_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname3_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname4_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname5_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname6_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname7_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname8_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname9_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname0_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname10_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname11_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname12_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname13_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname14_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname15_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname16_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname17_gfx942.s | 22 +- ...as_GG_AS_SAV_UserArgs_shortname18_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname1_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname2_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname3_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname4_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname5_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname6_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname7_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname8_gfx942.s | 22 +- ...ias_GG_AS_SAV_UserArgs_shortname9_gfx942.s | 22 +- ...256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s | 94374 ++++++++-------- ...28_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s | 22 +- 85 files changed, 48961 insertions(+), 47261 deletions(-) diff --git a/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s b/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s index b89fc28ae9..77f838e984 100644 --- a/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s @@ -63,7 +63,27 @@ custom.config: Activation: True UseScaleAlphaVec: 1 SupportUserArgs: False - MatrixInstruction: [16, 16, 16, 1, 1, 2,1, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 128 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s index 3674c844c2..0c70be040e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s @@ -59,7 +59,27 @@ custom.config: UseBias: 1 Activation: True UseScaleAlphaVec: 1 - MatrixInstruction: [16, 16, 16, 1, 1, 8,1, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [8, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 32 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s index 6e87f5aca6..4ac5825587 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s @@ -59,7 +59,27 @@ custom.config: UseBias: 1 Activation: True UseScaleAlphaVec: 1 - MatrixInstruction: [16, 16, 16, 1, 1, 2,2, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 2] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 64 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s index 0c83d6a61c..f4406dfa26 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s index d0f986a3f7..7ef9df1b0c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s index cd19ced94f..ab4c16ff21 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s index 8491de54b8..bdff04b446 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s index 6271f36e14..981f775192 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s index 297a1458f4..fed7525a1e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s index bc94ba39f8..cd26c62fe1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s index 3ebbdfa083..a633806eaa 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,10, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 10] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s index 1f0994c757..c075bcc0a4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s index 0cf92d8ea3..79afec6cf8 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s index 24ab9a9c50..96bf1a55fd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s index 66541fa0dc..8df5324deb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s index df3b5ef4b6..e5d308d026 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s index c59cd4b3c7..b56e55e18e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s index ec0b7fb75a..b3bdc5ebf0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s index 0931e6a7c7..fcc84176f1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s index 7321119941..c82e981471 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s index 344361d4cf..28a3fd19cd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s index e007147706..ceac6d116c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s index 197890e178..43bcd13213 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s index cc6f743af0..af61cb1e79 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s index cba2a8b539..98997d7b19 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s index 94156abc40..f0fc41a8e6 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s index 1016ce74a1..ea0301810d 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s index 3c388c6231..86a3300594 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s index 6066506b6f..be0bbe9fe4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s index 28cdb3dd46..63ac8fe345 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s index 92e9a4d8ac..bccdbc3383 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s index 1f3443996c..ae7432f8b2 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,10, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 10] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s index e6c2460ae4..90b1228119 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s index 513376dcac..3d4e7688a1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s index 407af1a4dc..7814e350a9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s index 6414e7c8f4..ec8bebf7f6 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s index 916cd5d59e..d51831eadb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s index 1c5e1faa88..82273f6ab3 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s index 5b95c072b8..1a44086769 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s index e179e481ba..d70720e24f 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s index ed247a1410..427eaec8f9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s index fd84446701..46e3e1c1ab 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s index ca184cda90..12a72a3edf 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s index 4e55195289..43b7698221 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s index 4943104af3..16c2974b35 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s index 8e2409bcda..a4d5590ff4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s index 2f9ee080b7..e1aae690af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s index 282cb330d7..eed0581627 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s index 9db4a1cbed..72ca30efb3 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s index 9786a5b756..08506c6223 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s index 580fd6577a..ffec6819fd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s index da2df8e691..5c5253cde0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s index 64734b241a..2be6c86c66 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s index 73ce86d687..013050f3fb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s index 69360d2ddf..df228c0923 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s index 326ac4eeb3..b3c9839640 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s index 2e22f24983..651040d735 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s index 5da71a8c30..d4c3e21c64 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s index 43b9c3244e..8ba8ce2d80 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s index 7aef4885fc..d6b0adf06c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s index 85baa5575f..d0e79e9909 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s index f054ea8a09..da6505fd69 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s index 6e40f868e7..a198f6a5af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s index 4eee79eaf3..0956cb45cc 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s index ba6a788e2e..7a2d3efadd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s index 7f4cc981a8..29e5245b9e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s index 85af1877f6..2b8e64eabc 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s index 052ebf3fbd..c9b2b475bb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s index afe4d304cd..1acd754b04 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s index 1dbfd0b224..31f955b65c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s index 7032dcd8f5..a7ef387537 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s index 166d0e4c9d..2ebc5b41c0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s index fb2216b86d..32260f7145 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s index ba9e5083a6..64035df006 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s index 259bc85cea..dd130c51c0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s index 23b5ef82bd..82edeb7b7d 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s index 337bbbf56a..6a37aecff5 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s index 421c90dc87..5cddea6d14 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s index df6b7e09ec..f6ae123378 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s index bd6113c409..26461bb12c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s index d763c33edb..db27d906b9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s index eba66bad78..b1d599fc72 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s index 878991f11d..52d55ad6b8 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s index c3f9640e04..50d91dc776 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s @@ -1,47177 +1,47197 @@ - -/******************************************/ -/* Begin Kernel */ -/******************************************/ -.amdgcn_target "amdgcn-amd-amdhsa--gfx942" -.text -.protected Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA -.globl Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA -.p2align 8 -.type Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA,@function -.section .rodata,#alloc -.p2align 6 -.amdhsa_kernel Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA - .amdhsa_user_sgpr_kernarg_segment_ptr 1 - .amdhsa_accum_offset 256 // accvgpr offset - .amdhsa_next_free_vgpr 512 // vgprs - .amdhsa_next_free_sgpr 84 // sgprs - .amdhsa_group_segment_fixed_size 40960 // lds bytes - .amdhsa_private_segment_fixed_size 0 - .amdhsa_system_sgpr_workgroup_id_x 1 - .amdhsa_system_sgpr_workgroup_id_y 1 - .amdhsa_system_sgpr_workgroup_id_z 1 - .amdhsa_system_vgpr_workitem_id 0 - .amdhsa_float_denorm_mode_32 3 - .amdhsa_float_denorm_mode_16_64 3 - .amdhsa_user_sgpr_count 13 - .amdhsa_user_sgpr_kernarg_preload_length 11 - .amdhsa_user_sgpr_kernarg_preload_offset 0 -.end_amdhsa_kernel -.text -/* Num VGPR =256 */ -/* Num AccVGPR=256 */ -/* Num SGPR =84 */ - -/******************************************/ -/* Optimizations and Config: */ -/******************************************/ -/* ThreadTile= 16 x 16 */ -/* SubGroup= 16 x 16 */ -/* VectorWidthA=4 */ -/* VectorWidthB=1 */ -/* GlobalReadVectorWidthA=8, GlobalReadVectorWidthB=8 */ -/* DirectToLdsA=False */ -/* DirectToLdsB=False */ -/* UseSgprForGRO=1 */ -.amdgpu_metadata ---- -custom.config: - ProblemType: - OperationType: GEMM - DataType: b - DestDataType: b - ComputeDataType: s - HighPrecisionAccumulate: True - TransposeA: True - TransposeB: False - UseBias: 1 - Activation: True - UseScaleAlphaVec: 1 - UseBeta: True - Batched: True - GroupedGemm: False - SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4, 16, 4, 1] - 1LDSBuffer: 1 - ScheduleIterAlg: 3 - DepthU: 64 - GlobalReadVectorWidthA: 8 - GlobalReadVectorWidthB: 8 - AssertFree0ElementMultiple: 1 - AssertFree1ElementMultiple: 1 - AssertSummationElementMultiple: 1 - NoReject: True - InternalSupportParams: - KernArgsVersion: 0 - SupportUserGSU: True - SupportCustomWGM: True - SupportCustomStaggerU: True - UseUniversalArgs: True -amdhsa.version: - - 1 - - 1 -amdhsa.kernels: - - .name: Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA - .symbol: 'Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.kd' - .language: OpenCL C - .language_version: - - 2 - - 0 - .args: - - .name: SizesFree0 - .size: 4 - .offset: 0 - .value_kind: by_value - .value_type: u32 - - .name: SizesFree1 - .size: 4 - .offset: 4 - .value_kind: by_value - .value_type: u32 - - .name: SizesFree2 - .size: 4 - .offset: 8 - .value_kind: by_value - .value_type: u32 - - .name: SizesSum0 - .size: 4 - .offset: 12 - .value_kind: by_value - .value_type: u32 - - .name: Gemm info - .size: 4 - .offset: 16 - .value_kind: by_value - .value_type: u32 - - .name: kernel info - .size: 4 - .offset: 20 - .value_kind: by_value - .value_type: u32 - - .name: D - .size: 8 - .offset: 24 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: C - .size: 8 - .offset: 32 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: A - .size: 8 - .offset: 40 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: B - .size: 8 - .offset: 48 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: strideD0 - .size: 4 - .offset: 56 - .value_kind: by_value - .value_type: u32 - - .name: strideD1 - .size: 4 - .offset: 60 - .value_kind: by_value - .value_type: u32 - - .name: strideC0 - .size: 4 - .offset: 64 - .value_kind: by_value - .value_type: u32 - - .name: strideC1 - .size: 4 - .offset: 68 - .value_kind: by_value - .value_type: u32 - - .name: strideA0 - .size: 4 - .offset: 72 - .value_kind: by_value - .value_type: u32 - - .name: strideA1 - .size: 4 - .offset: 76 - .value_kind: by_value - .value_type: u32 - - .name: strideB0 - .size: 4 - .offset: 80 - .value_kind: by_value - .value_type: u32 - - .name: strideB1 - .size: 4 - .offset: 84 - .value_kind: by_value - .value_type: u32 - - .name: alpha - .size: 4 - .offset: 88 - .value_kind: by_value - .value_type: f32 - - .name: beta - .size: 4 - .offset: 92 - .value_kind: by_value - .value_type: f32 - - .name: AddressScaleAlphaVec - .size: 8 - .offset: 96 - .value_kind: global_buffer - .value_type: f32 - .address_space: generic - - .name: bias - .size: 8 - .offset: 104 - .value_kind: global_buffer - .value_type: void - .address_space: generic - - .name: biasType - .size: 4 - .offset: 112 - .value_kind: by_value - .value_type: u32 - - .name: StrideBias - .size: 4 - .offset: 116 - .value_kind: by_value - .value_type: u32 - - .name: activationAlpha - .size: 4 - .offset: 120 - .value_kind: by_value - .value_type: f32 - - .name: activationBeta - .size: 4 - .offset: 124 - .value_kind: by_value - .value_type: f32 - - .name: activationType - .size: 4 - .offset: 128 - .value_kind: by_value - .value_type: u32 - .group_segment_fixed_size: 40960 - .kernarg_segment_align: 8 - .kernarg_segment_size: 136 - .max_flat_workgroup_size: 256 - .private_segment_fixed_size: 0 - .sgpr_count: 84 - .sgpr_spill_count: 0 - .vgpr_count: 256 - .vgpr_spill_count: 0 - .wavefront_size: 64 -... -.end_amdgpu_metadata -Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA: -label_ASM_Start: /// Main body of the asm kernel - -/* Magic div and mod functions */ -.macro V_MAGIC_DIV dstIdx:req dividend:req magicNumber:req magicShift:req magicA:req - v_mul_hi_u32 v[\dstIdx+1] \dividend \magicNumber - v_mul_lo_u32 v[\dstIdx+0] \dividend \magicA - v_add_u32 v[\dstIdx+0] v[\dstIdx+0] v[\dstIdx+1] - v_lshrrev_b32 v[\dstIdx+0] \magicShift v[\dstIdx+0] -.endm - -/******************************************/ -/* VGPR Assignments */ -/******************************************/ -/* ValuC range: [0-0), serializedStore enabled */ -.set vgprValuC, 0 -/* ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx */ -.set vgprValuA_X0_I0_0, 0 -.set vgprValuA_X2_I0_0, 16 -.set vgprValuB_X0_I0, 32 -.set vgprValuB_X2_I0, 96 -.set vgprLocalWriteAddrA, 160 -.set vgprLocalWriteAddrB, 161 -.set vgprGlobalReadOffsetA, 162 -.set vgprGlobalReadOffsetB, 163 -.set vgprG2LB, 164 -.set vgprValuA_X0_I0_1, 196 -.set vgprValuA_X2_I0_1, 212 -.set vgprLocalReadAddrA, 228 -.set vgprLocalReadAddrB, 229 -.set vgprSerial, 230 - -/******************************************/ -/* SGPR Assignments */ -/******************************************/ -.set sgprKernArgAddress, 0 -.set sgprWorkGroup0, 2 -.set sgprWorkGroup1, 3 -.set sgprWorkGroup2, 4 -.set sgprArgType, 5 -.set sgprGSUSumIdx, 6 -.set sgprGSULog2BpeC, 8 -.set sgprGSULog2BpeD, 9 -.set sgprStaggerU, 10 -.set sgprWGM, 11 -.set sgprLoopCounterL, 12 -.set sgprOrigLoopCounter, 13 -.set sgprSrdD, 16 -.set sgprSrdC, 20 -.set sgprNumWorkGroups0, 14 -.set sgprNumWorkGroups1, 15 -.set sgprSizesFree, 24 -.set sgprSizesSum, 27 -.set sgprAddressD, 28 -.set sgprAddressC, 30 -.set sgprAddressA, 32 -.set sgprAddressB, 34 -.set sgprStridesD, 36 -.set sgprStridesC, 38 -.set sgprStridesA, 40 -.set sgprStridesB, 42 -.set sgprAlpha, 44 -.set sgprBeta, 45 -.set sgprGSU, 46 - -/* Size Assignments */ -.set sgprSizeI, sgprSizesFree+0 -.set sgprSizeJ, sgprSizesFree+1 -.set sgprSizeK, sgprSizesFree+2 -.set sgprSizeL, sgprSizesSum+0 - -/* Stride Assignments */ -.set constStrideD0I, 1 -.set sgprStrideD1J, sgprStridesD+0 -.set sgprStrideDK, sgprStridesD+1 -.set constStrideC0I, 1 -.set sgprStrideC1J, sgprStridesC+0 -.set sgprStrideCK, sgprStridesC+1 -.set constStrideAL, 1 -.set sgprStrideA0I, sgprStridesA+0 -.set sgprStrideAK, sgprStridesA+1 -.set constStrideBL, 1 -.set sgprStrideB1J, sgprStridesB+0 -.set sgprStrideBK, sgprStridesB+1 - -.set MT0, 256 -.set MT1, 256 -.set DepthU, 64 -.set BpeA, 2 -.set BpeALog2, 1 -.set BpeB, 2 -.set BpeBLog2, 1 -.set BpeAGR, 2 -.set BpeAGRLog2, 1 -.set BpeBGR, 2 -.set BpeBGRLog2, 1 -/* Number of elements to shift-left SRD */ -.set SrdShiftLeftA, 8 -.set SrdShiftLeftB, 8 -/* 2GB limit - set offsets to -1 to exceed this and clamp */ -.set BufferLimit, 0xffffffff -.set BufferOOB, 0x80000000 - -/******************************************/ -/* Bits 127:96 of SRD. */ -/* hex: 0x00020000 */ -/* dst_sel_x (3b): 0 */ -/* dst_sel_y (3b): 0 */ -/* dst_sel_z (3b): 0 */ -/* dst_sel_w (3b): 0 */ -/* num_format (3b): 0 */ -/* data_format (4b): 4 */ -/* user_vm_enable (1b): 0 */ -/* user_vm_mode (1b): 0 */ -/* index_stride (2b): 0 */ -/* add_tid_enable (1b): 0 */ -/* _unusedA (3b): 0 */ -/* nv (1b): 0 */ -/* _unusedB (2b): 0 */ -/* type (2b): 0 */ -/******************************************/ -.set Srd127_96, 0x00020000 - -/* Global Offset A */ -.macro GLOBAL_OFFSET_A vgprAddr:req vgprTmp:req - v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) - v_and_b32 v[\vgprAddr+0], 15, v[\vgprTmp+0] // 1. M offset: mIdx = wtid % MI_M(16) - v_mul_lo_u32 v[\vgprAddr+0], s[sgprStrideA0I], v[\vgprAddr+0] // 1. M offset: mOffset = mIdx * mStride(k) - v_lshlrev_b32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) - v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) - v_lshrrev_b32 v[\vgprTmp+0], 4, v[\vgprTmp+0] // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) - v_lshlrev_b32 v[\vgprTmp+0], 0x3, v[\vgprTmp+0] // 5. K offset: lrKOffset = kIdx * mStride(8) - v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 6. offset in wave: lrOffset = bnOffset + lrKOffset - v_lshrrev_b32 v[\vgprTmp+0], 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) - v_and_b32 v[\vgprTmp+0], 3, v[\vgprTmp+0] // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) - v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideA0I], v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) - v_lshlrev_b32 v[\vgprTmp+0], 0x6, v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) - v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 7. final local read offset: flrOffset = lrOffset + WOffset - v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift - v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element -.endm - -/* Global Offset B */ -.macro GLOBAL_OFFSET_B vgprAddr:req vgprOffsetL:req vgprOffset1J:req vgprTmp:req - v_mul_lo_u32 v[\vgprTmp+0] s[sgprStrideB1J] v[\vgprOffset1J] // mul d1 lower - v_add_co_u32 v[\vgprAddr+0] vcc v[\vgprOffsetL] v[\vgprTmp+0] // accumulate K lower - v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift - v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element -.endm - -/* Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor; */ -.macro DYNAMIC_VECTOR_DIVIDE vQuotient vRemainder vDividend vDivisor vTmp0 vTmp1 sTmp - v_cvt_f32_u32 v[\vQuotient] v[\vDivisor] - v_rcp_f32 v[\vQuotient] v[\vQuotient] - v_mul_f32 v[\vQuotient] 0x4f800000 v[\vQuotient] - v_cvt_u32_f32 v[\vQuotient] v[\vQuotient] - v_mul_lo_u32 v[\vRemainder] v[\vDivisor] v[\vQuotient] - v_mul_hi_u32 v[\vTmp0] v[\vDivisor] v[\vQuotient] - v_sub_co_u32 v[\vTmp1] vcc 0x0 v[\vRemainder] - v_cmp_ne_i32 s[\sTmp:\sTmp+1] 0x0 v[\vTmp0] - v_cndmask_b32 v[\vRemainder] v[\vTmp1] v[\vRemainder] s[\sTmp:\sTmp+1] - v_mul_hi_u32 v[\vRemainder] v[\vRemainder] v[\vQuotient] - v_sub_co_u32 v[\vTmp0] vcc v[\vQuotient] v[\vRemainder] - v_add_co_u32 v[\vQuotient] vcc v[\vQuotient] v[\vRemainder] - v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vTmp0] s[\sTmp:\sTmp+1] - v_mul_hi_u32 v[\vQuotient] v[\vQuotient] v[\vDividend] - v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] - v_sub_co_u32 v[\vTmp0] vcc v[\vDividend] v[\vRemainder] - v_cmp_ge_u32 s[\sTmp:\sTmp+1] v[\vDividend] v[\vRemainder] - v_add_co_u32 v[\vRemainder] vcc 0x1 v[\vQuotient] - v_add_co_u32 v[\vTmp1] vcc -1 v[\vQuotient] - v_cmp_le_u32 vcc v[\vDivisor] v[\vTmp0] - s_and_b64 vcc s[\sTmp:\sTmp+1] vcc - v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vRemainder] vcc - v_cndmask_b32 v[\vQuotient] v[\vTmp1] v[\vQuotient] s[\sTmp:\sTmp+1] - v_cmp_ne_i32 vcc 0x0 v[\vDivisor] - v_cndmask_b32 v[\vQuotient] -1 v[\vQuotient] vcc // final result - v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] - v_sub_co_u32 v[\vRemainder] vcc v[\vDividend] v[\vRemainder] // final result -.endm - -/******************************************/ -/* Allocate Resources */ -/******************************************/ - -/* Load num of Gemms */ -s_load_dword s47, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 - -/* Load GSU data */ -s_load_dword s[sgprGSU], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x4 -s_waitcnt lgkmcnt(0) -s_lshr_b32 s48, s47, 0x1e // Get arg type -s_and_b32 s47, 0x3fffffff, s47 // Get nums of gemm -s_cmp_eq_u32 s48, 0 // Is kernel args -s_cbranch_scc0 label_HBMArgs -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 -s_load_dwordx4 s[40:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x40 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_waitcnt lgkmcnt(0) -s_branch label_LoadArgsEnd -label_HBMArgs: - -/* Load address of kernel arguments */ -s_load_dwordx2 s[sgprKernArgAddress:sgprKernArgAddress+1], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8 -s_waitcnt lgkmcnt(0) // wait for args to load -label_LoadArgsEnd: -s_branch label_common_kernel_entry - -/* pad 41 snops to satisfy 0x100 code size for Preload Backward Compatibility Prologue */ -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -label_Preload_Offset_Start: -s_and_b32 s47, 0x3fffffff, s2 // Get nums of gemm -s_lshr_b32 s48, s2, 0x1e // Get arg type -s_mov_b32 s[sgprGSU], s3 // Preload internal args -s_cmp_eq_u32 s48, 0 // Is kernel args -s_cbranch_scc0 label_Preload_HBMArgs -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dword s33, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x24 -s_load_dwordx2 s[34:35], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x28 -s_load_dwordx8 s[36:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x30 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_mov_b32 s24, s4 // move preload data to correct sgpr -s_mov_b32 s25, s5 // move preload data to correct sgpr -s_mov_b32 s26, s6 // move preload data to correct sgpr -s_mov_b32 s27, s7 // move preload data to correct sgpr -s_mov_b32 s28, s8 // move preload data to correct sgpr -s_mov_b32 s29, s9 // move preload data to correct sgpr -s_mov_b32 s30, s10 // move preload data to correct sgpr -s_mov_b32 s31, s11 // move preload data to correct sgpr -s_mov_b32 s32, s12 // move preload data to correct sgpr -s_branch label_Preload_LoadArgsEnd -label_Preload_HBMArgs: -s_mov_b64 s[sgprKernArgAddress:sgprKernArgAddress+1], s[4:5] // Load address of kernel arguments -label_Preload_LoadArgsEnd: -label_common_kernel_entry: /// for both preload/non-preload common code -s_mov_b32 s[sgprWorkGroup0+0], s13 // restore workgroup id -s_mov_b32 s[sgprWorkGroup0+1], s14 // restore workgroup id -s_mov_b32 s[sgprWorkGroup0+2], s15 // restore workgroup id -s_and_b32 s[sgprWGM], s[sgprGSU], 0xff00 // Restore WGM -s_lshr_b32 s[sgprWGM], s[sgprWGM], 0x8 -s_and_b32 s[sgprStaggerU], s[sgprGSU], 0xffff0000 // Restore StaggerU related vars -s_lshr_b32 s[sgprStaggerU], s[sgprStaggerU], 0x10 -s_and_b32 s[sgprGSU], s[sgprGSU], 0xff // Restore GSU -s_mov_b32 s[sgprArgType], s48 -s_mov_b32 m0, 0xa000 // LDS clamp at 40960 bytes -v_mov_b32 v[vgprSerial], v0 // thread serial id -s_cmp_eq_u32 s48, 0 -s_cbranch_scc0 label_MultiGemm -/* init: add vgpr [0...160) to pool */ -/* init: add vgpr [0...0) to pool */ -/* init: add agpr [0...256) to pool */ - -/******************************************/ -/* Local Read Addresses */ -/******************************************/ - -/* local read addresses: tile assignments a/b */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset - -/* local read addresses: final offsets a */ -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 - -/* local read addresses: final offsets b */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 - -/* local read addresses: declare addresses a */ -/* N/A */ - -/* local read addresses: declare addresses b */ - -/******************************************/ -/* Local Write Addresses */ -/******************************************/ -/* LVCA = 8 */ -/* v1 = A-unroll = serial%LVCA */ -v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 -v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 -v_mov_b32 v4, v1 // copy for GlobalSplitU -/* LVCB = 8 */ -/* v3 = B-unroll = serial%LVCB */ -v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 -v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 -v_mov_b32 v5, v3 // copy for GlobalSplitU -/* lwaUnrollAssignmentA = v4 */ -/* lwaUnrollAssignmentB = v5 */ - -/* local write addresses: first offset a */ -v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 -v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 - -/* local write addresses: first offset b */ -v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 -v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 -v_mov_b32 v8, MT0 // set MT0 into sgpr -v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -v_mov_b32 v8, MT1 // set MT1 into sgpr -v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size -v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -s_nop 0 // 1 wait states -v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 -s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args -s_branch label_MultiGemmEnd -label_MultiGemm: - -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_IsExternalValid // branch if ArgType == 2 -s_mov_b32 s15, 124 -s_mul_i32 s54, s47, 4 -s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] -s_branch label_IsExternalValidEnd -label_IsExternalValid: -s_mov_b32 s15, 196 -s_mov_b32 s54, 0x0 -s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] -label_IsExternalValidEnd: - -/* Grouped Gemm:: prefetch 1 arg load */ -s_mov_b32 s14, 1 -s_mov_b32 s55, 0 -s_load_dwordx4 s[24:27], s[48:49], s54 -s_cmpk_eq_u32 s47, 1 // if gemm_count is 1? -s_cbranch_scc1 label_wgTable_noLoadLoop - -/* Grouped Gemm:: accumulate numTiles for each gemm */ -/* Grouped Gemm:: loop start */ -label_Loop_GemmCount: -s_waitcnt lgkmcnt(0) -s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 -s_and_b32 s50, 255, s24 // s50 = s24 % 256 -s_addc_u32 s52, s52, 0x0 -s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 -s_and_b32 s50, 255, s25 // s50 = s25 % 256 -s_addc_u32 s53, s53, 0x0 -s_mul_i32 s52, s52, s53 -s_mul_i32 s52, s52, s26 -s_mul_i32 s52, s52, s[sgprGSU] -s_add_u32 s55, s55, s52 -s_cmp_lt_u32 s[sgprWorkGroup0], s55 -s_cbranch_scc1 label_FOUND -s_add_u32 s54, s54, s15 -s_load_dwordx4 s[24:27], s[48:49], s54 -s_add_u32 s14, s14, 1 -s_cmp_lt_u32 s14, s47 -s_cbranch_scc1 label_Loop_GemmCount - -/* Grouped Gemm:: noLoadLoop */ -label_wgTable_noLoadLoop: -s_waitcnt lgkmcnt(0) -s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 -s_and_b32 s50, 255, s24 // s50 = s24 % 256 -s_addc_u32 s52, s52, 0x0 -s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 -s_and_b32 s50, 255, s25 // s50 = s25 % 256 -s_addc_u32 s53, s53, 0x0 -s_mul_i32 s52, s52, s53 -s_mul_i32 s52, s52, s26 -s_mul_i32 s52, s52, s[sgprGSU] -s_add_u32 s55, s55, s52 - -/* Grouped Gemm:: gemmIndex found */ -label_FOUND: -s_sub_u32 s49, s14, 1 -s_sub_u32 s48, s55, s52 -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalStruct // branch if ArgType == 2 - -/* Grouped Gemm: offset argument address to gemm */ -/* Grouped Gemm: offset address from wg_table_start to args_start */ -s_lshl2_add_u32 s[sgprKernArgAddress], s47, s[sgprKernArgAddress] -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 -/* Grouped Gemm: offset address from args_start to gemm_start */ -s_mul_i32 s49, s49, 124 -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_branch label_LoadExternalStructEnd -label_LoadExternalStruct: -/* Grouped Gemm: offset address from args_start to gemm_start */ -s_mul_i32 s49, s49, 196 -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 -s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 -s_load_dword s44, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -// Read Beta -s_load_dword s45, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x60 -label_LoadExternalStructEnd: -/* init: add vgpr [0...160) to pool */ -/* init: add vgpr [0...0) to pool */ -/* init: add agpr [0...256) to pool */ - -/******************************************/ -/* Local Read Addresses */ -/******************************************/ - -/* local read addresses: tile assignments a/b */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset - -/* local read addresses: final offsets a */ -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 - -/* local read addresses: final offsets b */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 - -/* local read addresses: declare addresses a */ -/* N/A */ - -/* local read addresses: declare addresses b */ - -/******************************************/ -/* Local Write Addresses */ -/******************************************/ -/* LVCA = 8 */ -/* v1 = A-unroll = serial%LVCA */ -v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 -v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 -v_mov_b32 v4, v1 // copy for GlobalSplitU -/* LVCB = 8 */ -/* v3 = B-unroll = serial%LVCB */ -v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 -v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 -v_mov_b32 v5, v3 // copy for GlobalSplitU -/* lwaUnrollAssignmentA = v4 */ -/* lwaUnrollAssignmentB = v5 */ - -/* local write addresses: first offset a */ -v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 -v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 - -/* local write addresses: first offset b */ -v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 -v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 -v_mov_b32 v8, MT0 // set MT0 into sgpr -v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -v_mov_b32 v8, MT1 // set MT1 into sgpr -v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size -v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -s_nop 0 // 1 wait states -v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 -s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args - -/* Early stop if N(SizeFreeJ) == 0 */ -s_cmp_eq_u32 s[sgprSizeJ], 0x0 -s_cbranch_scc0 label_NoEarlyStop_N0 -label_EarlyStop_if_N_is_0: -s_endpgm -label_NoEarlyStop_N0: - -/* Grouped Gemm: remap wg from 1D(idxWG012) to 3D(wg2,wg1,wg0) */ -/* wg2 = idxWG012 * smallMagicNumber(1/(numWG0*numWG1)) */ -s_mul_i32 s48, s[sgprNumWorkGroups0], s[sgprNumWorkGroups1] -s_mul_i32 s48, s48, s[sgprGSU] -v_cvt_f32_u32 v6, s48 // s48 = s[sgprWorkGroup0] / s48 -v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 -v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s48 -v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s48 -v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 -v_mul_u32_u24 v7, v6, s48 // s48 = s[sgprWorkGroup0] / s48 -v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s48 -v_cmpx_eq_u32 exec, v7, s48 // s48 = s[sgprWorkGroup0] / s48 -v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s48 -s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s48 -v_readfirstlane_b32 s48, v6 -s_mov_b32 s[sgprWorkGroup2], s48 -/* idxWG01 = idxWG012 - wg2 * numWG0 * numWG1 */ -s_mul_i32 s48, s[sgprNumWorkGroups1], s[sgprNumWorkGroups0] -s_mul_i32 s48, s48, s[sgprWorkGroup2] -s_mul_i32 s48, s48, s[sgprGSU] -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 -/* wg1 = idxWG01 * smallMagicNumber(1/numWG0) */ -v_cvt_f32_u32 v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_readfirstlane_b32 s48, v6 -s_mov_b32 s[sgprWorkGroup1], s48 -/* wg0 = idxWG01 - wg1 * numWG0 */ -s_mul_i32 s48, s[sgprWorkGroup1], s[sgprNumWorkGroups0] -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 - -/* Early stop if wg exceed */ -s_cmp_ge_u32 s[sgprWorkGroup2], s[sgprSizesFree+2] -s_cbranch_scc0 label_NoEarlyStop_wgExceed -label_EarlyStop_if_wg_exceed: -s_endpgm -label_NoEarlyStop_wgExceed: - -label_MultiGemmEnd: -.set sgprSrdA, 48 -.set sgprSrdB, 52 -.set sgprShadowLimitA, 56 -.set sgprShadowLimitB, 58 -.set sgprStaggerUIter, 47 -.set sgprWrapUA, 60 -.set sgprWrapUB, 62 -.set sgprGlobalReadIncsA, 64 -.set sgprGlobalReadIncsB, 65 -.set sgprScalarGlobalReadOffsetA, 66 -.set sgprScalarGlobalReadOffsetB, 73 -s_sub_u32 s[sgprAddressA+0], s[sgprAddressA+0], 16 // pre-pad to make room for possible pointer shift -s_subb_u32 s[sgprAddressA+1], s[sgprAddressA+1], 0 // pre-pad to make room for possible pointer shift -s_sub_u32 s[sgprAddressB+0], s[sgprAddressB+0], 16 // pre-pad to make room for possible pointer shift -s_subb_u32 s[sgprAddressB+1], s[sgprAddressB+1], 0 // pre-pad to make room for possible pointer shift - -/* Short circuit condition if Alpha == 0, then sumDims=0 */ -v_cmp_eq_f32 vcc, s[sgprAlpha], 0.0 // s[Alpha] == 0.0f ? -s_cbranch_vccz label_AlphaNonZero // branch if s[Alpha] != 0 -s_mov_b32 s[sgprSizesSum+0], 0x0 // Set summation dim=0 if Alpha == 0 -label_AlphaNonZero: - -/******************************************/ -/* Begin setupNewTile */ -/******************************************/ - -/* global read addresses: work-group */ -/* graWorkGroup mapping */ -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU // branch if GSU == 1 -// GSU-not-WGMapRR :nwg1 = (size1J + MT1J - 1) / MT1J; -v_cvt_f32_u32 v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cvt_f32_u32 v7, s[sgprWorkGroup1] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mul_f32 v6, v6, v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mul_u32_u24 v7, v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_sub_u32 v7, s[sgprWorkGroup1], v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cmpx_eq_u32 exec, v7, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_add_u32 v6, 1, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mov_b32 v7, 0 // s[sgprGSUSumIdx] = s[sgprWorkGroup1] % s[sgprGSU] -s_mov_b64 exec, -1 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_readfirstlane_b32 s[sgprWorkGroup1], v6 -v_readfirstlane_b32 s[sgprGSUSumIdx], v7 -s_mov_b32 s[sgprGSULog2BpeC], 1 -s_mov_b32 s[sgprGSULog2BpeD], 2 -s_branch label_GSU_End -label_GSU: -s_mov_b64 s[sgprGSUSumIdx:sgprGSUSumIdx+1], 0 // Set GSUSumIdx to 0 -s_mov_b32 s[sgprGSULog2BpeC], 1 -s_mov_b32 s[sgprGSULog2BpeD], 1 -label_GSU_End: -s_cmp_le_u32 s[sgprWGM], 1 // WGM <= 1 ? -s_cbranch_scc1 label_WGM // branch if WGM <= 1 -v_cvt_f32_u32 v6, s[sgprWGM] // WGM -v_rcp_iflag_f32 v6, v6 // WGM -v_cvt_f32_u32 v7, s[sgprWorkGroup1] // WGM -v_mul_f32 v6, v6, v7 // WGM -v_cvt_u32_f32 v6, v6 // WGM -v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM -v_sub_u32 v7, s[sgprWorkGroup1], v7 // WGM -v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM -v_add_u32 v6, 1, v6 // WGM -s_mov_b64 exec, -1 // WGM -v_readfirstlane_b32 s82, v6 -s_mul_i32 s83, s82, s[sgprWGM] // quotient * non-magic divisor -s_sub_u32 s83, s[sgprWorkGroup1], s83 // WorkGroup1=remainder -s_mul_i32 s83, s83, s[sgprNumWorkGroups0] // (wg1 % WGM)*nwg0 -s_add_u32 s83, s83, s[sgprWorkGroup0] // wgSerial = wg0 + (wg1 % WGM)*nwg0 -v_cvt_f32_u32 v6, s[sgprWGM] // WGM -v_rcp_iflag_f32 v6, v6 // WGM -v_cvt_f32_u32 v7, s[sgprNumWorkGroups1] // WGM -v_mul_f32 v6, v6, v7 // WGM -v_cvt_u32_f32 v6, v6 // WGM -v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM -v_sub_u32 v7, s[sgprNumWorkGroups1], v7 // WGM -v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM -v_add_u32 v6, 1, v6 // WGM -s_mov_b64 exec, -1 // WGM -v_readfirstlane_b32 s80, v6 -s_mul_i32 s81, s[sgprWGM], s80 // quotient * non-magic divisor -s_sub_u32 s81, s[sgprNumWorkGroups1], s81 // WorkGroup1=remainder -s_cmp_eq_u32 s81, 0 // remainder == 0 ? -s_cmov_b32 s81, s[sgprWGM] // remainder = WGM if remainder == 0 -s_cmp_ge_u32 s82, s80 // blockId >= numFullBlocks ? -s_cselect_b32 s80, s81, s[sgprWGM] -v_cvt_f32_u32 v6, s80 // s[sgprWorkGroup0] = s83 / s80 -v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 -v_cvt_f32_u32 v7, s83 // s[sgprWorkGroup0] = s83 / s80 -v_mul_f32 v6, v6, v7 // s[sgprWorkGroup0] = s83 / s80 -v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 -v_mul_u32_u24 v7, v6, s80 // s[sgprWorkGroup0] = s83 / s80 -v_sub_u32 v7, s83, v7 // s[sgprWorkGroup0] = s83 / s80 -v_cmpx_eq_u32 exec, v7, s80 // s[sgprWorkGroup0] = s83 / s80 -v_add_u32 v6, 1, v6 // s[sgprWorkGroup0] = s83 / s80 -v_mov_b32 v7, 0 // s[sgprWorkGroup1] = s83 % s80 -s_mov_b64 exec, -1 // s[sgprWorkGroup0] = s83 / s80 -v_readfirstlane_b32 s[sgprWorkGroup0], v6 -v_readfirstlane_b32 s[sgprWorkGroup1], v7 -s_mul_i32 s82, s82, s[sgprWGM] // blockId * WGM -s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s82 // wg1 += blockId * WGM -label_WGM: - -/* global read addresses: tile offset assignment a */ -/* graTileAssignmentA = v0 */ - -/* global read addresses: tile offset assignment b */ -/* graTileAssignmentB = v2 */ - -/* global read addresses: unroll assignment a */ -/* v1 */ - -/* global read addresses: unroll assignment b */ -/* v3 */ - -/* global read addresses: other free assignments */ -/* s[sgprWorkGroup2] */ - -/* global read addresses: tile offsets a */ - -/* global read addresses: tile offsets b */ - -/* global read addresses: unroll offsets a */ - -/* global read addresses: unroll offsets b */ - -/* global read addresses: final offsets a */ -GLOBAL_OFFSET_A vgprGlobalReadOffsetA+0, 6 -s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 32 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] - -/* global read addresses: final offsets b */ -GLOBAL_OFFSET_B vgprGlobalReadOffsetB+0, 3, 2, 6 // gROB_0_0_0_0 -s_mul_i32 s[sgprScalarGlobalReadOffsetB+0], s[sgprStrideB1J], 32 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+0], s[sgprScalarGlobalReadOffsetB+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+1], s[sgprStrideB1J], 64 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+1], s[sgprScalarGlobalReadOffsetB+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+2], s[sgprStrideB1J], 96 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+2], s[sgprScalarGlobalReadOffsetB+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+3], s[sgprStrideB1J], 128 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+3], s[sgprScalarGlobalReadOffsetB+3], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+4], s[sgprStrideB1J], 160 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+4], s[sgprScalarGlobalReadOffsetB+4], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+5], s[sgprStrideB1J], 192 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+5], s[sgprScalarGlobalReadOffsetB+5], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+6], s[sgprStrideB1J], 224 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+6], s[sgprScalarGlobalReadOffsetB+6], 0x1 // scalar offset *= bytes/element - -/* global read addresses: addresses a */ -/* max read offset = size[n] * stride[n-1] */ -s_mul_hi_u32 s83, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT -s_mul_i32 s82, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT -s_mul_hi_u32 s83, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride -s_mul_i32 s82, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride -s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart -s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart -s_mov_b32 s[sgprShadowLimitA+0], 1 // Init tensor size -s_mov_b32 s[sgprShadowLimitA+1], 0 // init tensor size -s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) -s_mul_hi_u32 s81, constStrideAL, s80 // stride x (size-1) -s_mul_i32 s80, constStrideAL, s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size -s_sub_u32 s80, s[sgprSizeI], 1 // (size-1) -s_mul_hi_u32 s81, s[sgprStrideA0I], s80 // stride x (size-1) -s_mul_i32 s80, s[sgprStrideA0I], s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s82 // sub tileStart -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s83 // sub tileStart -s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], 0x1 // Set limit to use bytes -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], 16 // extend limit for pre-pad -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], 0 // extend limit for pre-pad -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -s_mul_hi_u32 s81, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG -s_mul_i32 s80, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG -s_add_u32 s82, s82, s80 // accum wg term to tilestart -s_addc_u32 s83, s83, s81 // accum wg term to tilestart -s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE -s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s82 // SRD base = Address+ tileStart0 -s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s83 // SRD base = Address+ tileStart1 -s_mov_b32 s[sgprSrdA+3], Srd127_96 // Set bits 127_96 in SRD - -/* global read addresses: addresses b */ -/* max read offset = size[n] * stride[n-1] */ -s_mul_hi_u32 s83, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT -s_mul_i32 s82, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT -s_mul_hi_u32 s83, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride -s_mul_i32 s82, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride -s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart -s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart -s_mov_b32 s[sgprShadowLimitB+0], 1 // Init tensor size -s_mov_b32 s[sgprShadowLimitB+1], 0 // init tensor size -s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) -s_mul_hi_u32 s81, constStrideBL, s80 // stride x (size-1) -s_mul_i32 s80, constStrideBL, s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size -s_sub_u32 s80, s[sgprSizeJ], 1 // (size-1) -s_mul_hi_u32 s81, s[sgprStrideB1J], s80 // stride x (size-1) -s_mul_i32 s80, s[sgprStrideB1J], s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s82 // sub tileStart -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s83 // sub tileStart -s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], 0x1 // Set limit to use bytes -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], 16 // extend limit for pre-pad -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], 0 // extend limit for pre-pad -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -s_mul_hi_u32 s81, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG -s_mul_i32 s80, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG -s_add_u32 s82, s82, s80 // accum wg term to tilestart -s_addc_u32 s83, s83, s81 // accum wg term to tilestart -s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE -s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s82 // SRD base = Address+ tileStart0 -s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s83 // SRD base = Address+ tileStart1 -s_mov_b32 s[sgprSrdB+3], Srd127_96 // Set bits 127_96 in SRD -s_mul_i32 s80, s[sgprGSU], DepthU*BpeAGR -s_mov_b32 s[sgprGlobalReadIncsA+0], s80 // incrA (unrollIdx) - -/* global read addresses: increments b */ -s_mul_i32 s80, s[sgprGSU], DepthU*BpeBGR -s_mov_b32 s[sgprGlobalReadIncsB+0], s80 // incrB (unrollIdx) - -/* declare loop num iterations */ -s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], 6 // s[sgprLoopCounterL] = s[sgprSizesSum+0] / 64 -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_1 // branch if GSU == 1 -v_cvt_f32_u32 v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_rcp_iflag_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cvt_f32_u32 v1, s[sgprLoopCounterL] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mul_f32 v0, v0, v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cvt_u32_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mul_u32_u24 v1, v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_sub_u32 v1, s[sgprLoopCounterL], v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cmpx_eq_u32 exec, v1, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_add_u32 v0, 1, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mov_b32 v1, 0 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSU] -s_mov_b64 exec, -1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_readfirstlane_b32 s[sgprLoopCounterL], v0 -v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1 -s_add_u32 s80, 1, s[sgprLoopCounterL] // tmp<-numIterMyWg+ -s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx < numIterPerWgRemainder -s_cmov_b32 s[sgprLoopCounterL], s80 // numIterMyWg++ if needed -label_GSU_1: -s_mov_b32 s[sgprOrigLoopCounter], s[sgprLoopCounterL] // copy loop counter -s_and_b32 s82, s[sgprStaggerU], 0x1f00 -s_lshr_b32 s82, s82, 0x8 -s_and_b32 s83, s[sgprStaggerU], 0xe000 -s_and_b32 s[sgprStaggerU], s[sgprStaggerU], 0xff -s_mov_b32 s80, s[sgprStaggerU] // init staggerU -label_beginStaggerUIter: -s_lshl_b32 s81, s80, s82 // shift by StaggerUStride -s_cmp_ge_u32 s[sgprOrigLoopCounter], s81 // loopCount >= current shift Count -s_cbranch_scc1 label_endStaggerUIter // jump to end -s_lshr_b32 s80, s80, 1 // step down to smaller stagger -s_branch label_beginStaggerUIter // jump to begin -label_endStaggerUIter: -s_sub_u32 s81, s80, 1 // staggerU mask -s_cmp_ge_u32 s80, 1 // if current staggerU >= 1 -s_cselect_b32 s[sgprStaggerUIter], s81, 0 // set Mask -s_cmp_eq_u32 s83, 0x0 -s_cbranch_scc1 label_StaggerUMapping_1 -s_mov_b32 s80, s[sgprWorkGroup0] -s_branch label_staggerInputEnd -label_StaggerUMapping_1: -s_cmp_eq_u32 s83, 0x2000 -s_cbranch_scc1 label_StaggerUMapping_2 -s_mov_b32 s80, s[sgprWorkGroup1] -s_branch label_staggerInputEnd -label_StaggerUMapping_2: -s_cmp_eq_u32 s83, 0x4000 -s_cbranch_scc1 label_StaggerUMapping_3 -s_mov_b32 s80, -0x1 -s_branch label_staggerInputEnd -label_StaggerUMapping_3: -s_cmp_eq_u32 s83, 0x6000 -s_cbranch_scc1 label_StaggerUMapping_4 -s_mul_i32 s81, s[sgprNumWorkGroups0], s[sgprWorkGroup1] -s_add_u32 s80, s80, s81 -s_add_u32 s80, s80, s[sgprWorkGroup0] -s_branch label_staggerInputEnd -label_StaggerUMapping_4: -s_cmp_eq_u32 s83, 0x8000 -s_cbranch_scc1 label_staggerInputEnd -s_mov_b32 s80, -0x1 -s_branch label_staggerInputEnd -label_staggerInputEnd: -s_and_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s80 // Compute actual stagger start for this tile -s_lshl_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s82 // shift by StaggerUStride - -/* SRDs += (StaggerUIter) * GlobalReadIncsA+0 */ -s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset -s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset -s_mul_hi_i32 s[sgprWrapUA+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop -s_mul_i32 s[sgprWrapUA+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop -s_sub_u32 s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0], s[sgprWrapUA+0] // remove one iteration -s_subb_u32 s[sgprWrapUA+1], 0, s[sgprWrapUA+1] // remove one iteration -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* SRDs += (StaggerUIter) * GlobalReadIncsB+0 */ -s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset -s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset -s_mul_hi_i32 s[sgprWrapUB+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop -s_mul_i32 s[sgprWrapUB+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop -s_sub_u32 s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0], s[sgprWrapUB+0] // remove one iteration -s_subb_u32 s[sgprWrapUB+1], 0, s[sgprWrapUB+1] // remove one iteration -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -s_add_u32 s[sgprStaggerUIter], s[sgprStaggerUIter], 2 // Subtract (PGR-1); StaggerUIter now contains target iteration to wrap -/* local read addresses: init pointers a */ - -/* localReadInitPointers */ -/* local read addresses: init pointers b */ - -/* localReadInitPointers */ - -/* prefetch: global -> local */ -s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? -s_cbranch_scc1 label_ShadowInitStart // skip to ShadowInitStart iter b/c numIter==0 -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+0:vgprValuA_X2_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+4:vgprValuA_X2_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+8:vgprValuA_X2_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+12:vgprValuA_X2_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 - -/* global read inc A loopL */ -s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) -s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* global read inc B loopL */ -s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) -s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 - -/******************************************/ -/* End setupNewTile */ -/******************************************/ -label_ShadowInitStart: -s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdD+2], 0x80000000 -s_mov_b32 s[sgprSrdD+3], Srd127_96 // Set bits 127_96 in post-loop SRD - -s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdC+2], 0x80000000 -s_mov_b32 s[sgprSrdC+3], Srd127_96 // Set bits 127_96 in post-loop SRD - - -s_mul_i32 s82, MT1, s[sgprWorkGroup1] // <- wg1*MT1 -s_mul_hi_u32 s81, s82, s[sgprStrideC1J] // ScaleC s82 by Stride -s_mul_i32 s80, s82, s[sgprStrideC1J] // ScaleC s82 by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe -s_add_u32 s[sgprSrdC+0], s[sgprAddressC+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdC+1], s[sgprAddressC+1], s81 // add hi to SRD -s_mul_hi_u32 s81, s82, s[sgprStrideD1J] // ScaleD s82 by Stride -s_mul_i32 s80, s82, s[sgprStrideD1J] // ScaleD s82 by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprAddressD+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprAddressD+1], s81 // add hi to SRD - -s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride -s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s81 // add hi to SRD -s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride -s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi to SRD - -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_2 // branch if GSU == 1 -// GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s -s_mul_hi_u32 s81, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 -s_mul_i32 s80, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 -s_sub_u32 s82, s[sgprSizesFree+1], 1 // Free1 -s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free1 -s_mul_hi_u32 s83, s82, s[sgprStrideC1J] // Free1 -s_mul_i32 s82, s82, s[sgprStrideC1J] // Free1 -s_add_u32 s80, s80, s82 // Free1 -s_addc_u32 s81, s81, s83 // Free1 -s_sub_u32 s82, s[sgprSizesFree+2], 1 // Free2 -s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free2 -s_mul_hi_u32 s83, s82, s[sgprStrideCK] // Free2 -s_mul_i32 s82, s82, s[sgprStrideCK] // Free2 -s_add_u32 s80, s80, s82 // Free2 -s_addc_u32 s81, s81, s83 // Free2 -s_lshl_b64 s[80:81], s[80:81], 2 // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo GSU offset to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi GSU offset to SRD -label_GSU_2: -.set sgprGSULog2BpeC, UNDEF - -/* initC: remove ValuC vgpr buffer [0...0) from pool */ - -/* initC: remove acc vgpr buffer [0...256) from pool */ - -/* initC: remove ValuA/B vgpr buffer [0...160) from pool */ -v_accvgpr_write acc0, 0x0 // initC -v_accvgpr_write acc1, 0x0 // initC -v_accvgpr_write acc2, 0x0 // initC -v_accvgpr_write acc3, 0x0 // initC -v_accvgpr_write acc4, 0x0 // initC -v_accvgpr_write acc5, 0x0 // initC -v_accvgpr_write acc6, 0x0 // initC -v_accvgpr_write acc7, 0x0 // initC -v_accvgpr_write acc8, 0x0 // initC -v_accvgpr_write acc9, 0x0 // initC -v_accvgpr_write acc10, 0x0 // initC -v_accvgpr_write acc11, 0x0 // initC -v_accvgpr_write acc12, 0x0 // initC -v_accvgpr_write acc13, 0x0 // initC -v_accvgpr_write acc14, 0x0 // initC -v_accvgpr_write acc15, 0x0 // initC -v_accvgpr_write acc16, 0x0 // initC -v_accvgpr_write acc17, 0x0 // initC -v_accvgpr_write acc18, 0x0 // initC -v_accvgpr_write acc19, 0x0 // initC -v_accvgpr_write acc20, 0x0 // initC -v_accvgpr_write acc21, 0x0 // initC -v_accvgpr_write acc22, 0x0 // initC -v_accvgpr_write acc23, 0x0 // initC -v_accvgpr_write acc24, 0x0 // initC -v_accvgpr_write acc25, 0x0 // initC -v_accvgpr_write acc26, 0x0 // initC -v_accvgpr_write acc27, 0x0 // initC -v_accvgpr_write acc28, 0x0 // initC -v_accvgpr_write acc29, 0x0 // initC -v_accvgpr_write acc30, 0x0 // initC -v_accvgpr_write acc31, 0x0 // initC -v_accvgpr_write acc32, 0x0 // initC -v_accvgpr_write acc33, 0x0 // initC -v_accvgpr_write acc34, 0x0 // initC -v_accvgpr_write acc35, 0x0 // initC -v_accvgpr_write acc36, 0x0 // initC -v_accvgpr_write acc37, 0x0 // initC -v_accvgpr_write acc38, 0x0 // initC -v_accvgpr_write acc39, 0x0 // initC -v_accvgpr_write acc40, 0x0 // initC -v_accvgpr_write acc41, 0x0 // initC -v_accvgpr_write acc42, 0x0 // initC -v_accvgpr_write acc43, 0x0 // initC -v_accvgpr_write acc44, 0x0 // initC -v_accvgpr_write acc45, 0x0 // initC -v_accvgpr_write acc46, 0x0 // initC -v_accvgpr_write acc47, 0x0 // initC -v_accvgpr_write acc48, 0x0 // initC -v_accvgpr_write acc49, 0x0 // initC -v_accvgpr_write acc50, 0x0 // initC -v_accvgpr_write acc51, 0x0 // initC -v_accvgpr_write acc52, 0x0 // initC -v_accvgpr_write acc53, 0x0 // initC -v_accvgpr_write acc54, 0x0 // initC -v_accvgpr_write acc55, 0x0 // initC -v_accvgpr_write acc56, 0x0 // initC -v_accvgpr_write acc57, 0x0 // initC -v_accvgpr_write acc58, 0x0 // initC -v_accvgpr_write acc59, 0x0 // initC -v_accvgpr_write acc60, 0x0 // initC -v_accvgpr_write acc61, 0x0 // initC -v_accvgpr_write acc62, 0x0 // initC -v_accvgpr_write acc63, 0x0 // initC -v_accvgpr_write acc64, 0x0 // initC -v_accvgpr_write acc65, 0x0 // initC -v_accvgpr_write acc66, 0x0 // initC -v_accvgpr_write acc67, 0x0 // initC -v_accvgpr_write acc68, 0x0 // initC -v_accvgpr_write acc69, 0x0 // initC -v_accvgpr_write acc70, 0x0 // initC -v_accvgpr_write acc71, 0x0 // initC -v_accvgpr_write acc72, 0x0 // initC -v_accvgpr_write acc73, 0x0 // initC -v_accvgpr_write acc74, 0x0 // initC -v_accvgpr_write acc75, 0x0 // initC -v_accvgpr_write acc76, 0x0 // initC -v_accvgpr_write acc77, 0x0 // initC -v_accvgpr_write acc78, 0x0 // initC -v_accvgpr_write acc79, 0x0 // initC -v_accvgpr_write acc80, 0x0 // initC -v_accvgpr_write acc81, 0x0 // initC -v_accvgpr_write acc82, 0x0 // initC -v_accvgpr_write acc83, 0x0 // initC -v_accvgpr_write acc84, 0x0 // initC -v_accvgpr_write acc85, 0x0 // initC -v_accvgpr_write acc86, 0x0 // initC -v_accvgpr_write acc87, 0x0 // initC -v_accvgpr_write acc88, 0x0 // initC -v_accvgpr_write acc89, 0x0 // initC -v_accvgpr_write acc90, 0x0 // initC -v_accvgpr_write acc91, 0x0 // initC -v_accvgpr_write acc92, 0x0 // initC -v_accvgpr_write acc93, 0x0 // initC -v_accvgpr_write acc94, 0x0 // initC -v_accvgpr_write acc95, 0x0 // initC -v_accvgpr_write acc96, 0x0 // initC -v_accvgpr_write acc97, 0x0 // initC -v_accvgpr_write acc98, 0x0 // initC -v_accvgpr_write acc99, 0x0 // initC -v_accvgpr_write acc100, 0x0 // initC -v_accvgpr_write acc101, 0x0 // initC -v_accvgpr_write acc102, 0x0 // initC -v_accvgpr_write acc103, 0x0 // initC -v_accvgpr_write acc104, 0x0 // initC -v_accvgpr_write acc105, 0x0 // initC -v_accvgpr_write acc106, 0x0 // initC -v_accvgpr_write acc107, 0x0 // initC -v_accvgpr_write acc108, 0x0 // initC -v_accvgpr_write acc109, 0x0 // initC -v_accvgpr_write acc110, 0x0 // initC -v_accvgpr_write acc111, 0x0 // initC -v_accvgpr_write acc112, 0x0 // initC -v_accvgpr_write acc113, 0x0 // initC -v_accvgpr_write acc114, 0x0 // initC -v_accvgpr_write acc115, 0x0 // initC -v_accvgpr_write acc116, 0x0 // initC -v_accvgpr_write acc117, 0x0 // initC -v_accvgpr_write acc118, 0x0 // initC -v_accvgpr_write acc119, 0x0 // initC -v_accvgpr_write acc120, 0x0 // initC -v_accvgpr_write acc121, 0x0 // initC -v_accvgpr_write acc122, 0x0 // initC -v_accvgpr_write acc123, 0x0 // initC -v_accvgpr_write acc124, 0x0 // initC -v_accvgpr_write acc125, 0x0 // initC -v_accvgpr_write acc126, 0x0 // initC -v_accvgpr_write acc127, 0x0 // initC -v_accvgpr_write acc128, 0x0 // initC -v_accvgpr_write acc129, 0x0 // initC -v_accvgpr_write acc130, 0x0 // initC -v_accvgpr_write acc131, 0x0 // initC -v_accvgpr_write acc132, 0x0 // initC -v_accvgpr_write acc133, 0x0 // initC -v_accvgpr_write acc134, 0x0 // initC -v_accvgpr_write acc135, 0x0 // initC -v_accvgpr_write acc136, 0x0 // initC -v_accvgpr_write acc137, 0x0 // initC -v_accvgpr_write acc138, 0x0 // initC -v_accvgpr_write acc139, 0x0 // initC -v_accvgpr_write acc140, 0x0 // initC -v_accvgpr_write acc141, 0x0 // initC -v_accvgpr_write acc142, 0x0 // initC -v_accvgpr_write acc143, 0x0 // initC -v_accvgpr_write acc144, 0x0 // initC -v_accvgpr_write acc145, 0x0 // initC -v_accvgpr_write acc146, 0x0 // initC -v_accvgpr_write acc147, 0x0 // initC -v_accvgpr_write acc148, 0x0 // initC -v_accvgpr_write acc149, 0x0 // initC -v_accvgpr_write acc150, 0x0 // initC -v_accvgpr_write acc151, 0x0 // initC -v_accvgpr_write acc152, 0x0 // initC -v_accvgpr_write acc153, 0x0 // initC -v_accvgpr_write acc154, 0x0 // initC -v_accvgpr_write acc155, 0x0 // initC -v_accvgpr_write acc156, 0x0 // initC -v_accvgpr_write acc157, 0x0 // initC -v_accvgpr_write acc158, 0x0 // initC -v_accvgpr_write acc159, 0x0 // initC -v_accvgpr_write acc160, 0x0 // initC -v_accvgpr_write acc161, 0x0 // initC -v_accvgpr_write acc162, 0x0 // initC -v_accvgpr_write acc163, 0x0 // initC -v_accvgpr_write acc164, 0x0 // initC -v_accvgpr_write acc165, 0x0 // initC -v_accvgpr_write acc166, 0x0 // initC -v_accvgpr_write acc167, 0x0 // initC -v_accvgpr_write acc168, 0x0 // initC -v_accvgpr_write acc169, 0x0 // initC -v_accvgpr_write acc170, 0x0 // initC -v_accvgpr_write acc171, 0x0 // initC -v_accvgpr_write acc172, 0x0 // initC -v_accvgpr_write acc173, 0x0 // initC -v_accvgpr_write acc174, 0x0 // initC -v_accvgpr_write acc175, 0x0 // initC -v_accvgpr_write acc176, 0x0 // initC -v_accvgpr_write acc177, 0x0 // initC -v_accvgpr_write acc178, 0x0 // initC -v_accvgpr_write acc179, 0x0 // initC -v_accvgpr_write acc180, 0x0 // initC -v_accvgpr_write acc181, 0x0 // initC -v_accvgpr_write acc182, 0x0 // initC -v_accvgpr_write acc183, 0x0 // initC -v_accvgpr_write acc184, 0x0 // initC -v_accvgpr_write acc185, 0x0 // initC -v_accvgpr_write acc186, 0x0 // initC -v_accvgpr_write acc187, 0x0 // initC -v_accvgpr_write acc188, 0x0 // initC -v_accvgpr_write acc189, 0x0 // initC -v_accvgpr_write acc190, 0x0 // initC -v_accvgpr_write acc191, 0x0 // initC -v_accvgpr_write acc192, 0x0 // initC -v_accvgpr_write acc193, 0x0 // initC -v_accvgpr_write acc194, 0x0 // initC -v_accvgpr_write acc195, 0x0 // initC -v_accvgpr_write acc196, 0x0 // initC -v_accvgpr_write acc197, 0x0 // initC -v_accvgpr_write acc198, 0x0 // initC -v_accvgpr_write acc199, 0x0 // initC -v_accvgpr_write acc200, 0x0 // initC -v_accvgpr_write acc201, 0x0 // initC -v_accvgpr_write acc202, 0x0 // initC -v_accvgpr_write acc203, 0x0 // initC -v_accvgpr_write acc204, 0x0 // initC -v_accvgpr_write acc205, 0x0 // initC -v_accvgpr_write acc206, 0x0 // initC -v_accvgpr_write acc207, 0x0 // initC -v_accvgpr_write acc208, 0x0 // initC -v_accvgpr_write acc209, 0x0 // initC -v_accvgpr_write acc210, 0x0 // initC -v_accvgpr_write acc211, 0x0 // initC -v_accvgpr_write acc212, 0x0 // initC -v_accvgpr_write acc213, 0x0 // initC -v_accvgpr_write acc214, 0x0 // initC -v_accvgpr_write acc215, 0x0 // initC -v_accvgpr_write acc216, 0x0 // initC -v_accvgpr_write acc217, 0x0 // initC -v_accvgpr_write acc218, 0x0 // initC -v_accvgpr_write acc219, 0x0 // initC -v_accvgpr_write acc220, 0x0 // initC -v_accvgpr_write acc221, 0x0 // initC -v_accvgpr_write acc222, 0x0 // initC -v_accvgpr_write acc223, 0x0 // initC -v_accvgpr_write acc224, 0x0 // initC -v_accvgpr_write acc225, 0x0 // initC -v_accvgpr_write acc226, 0x0 // initC -v_accvgpr_write acc227, 0x0 // initC -v_accvgpr_write acc228, 0x0 // initC -v_accvgpr_write acc229, 0x0 // initC -v_accvgpr_write acc230, 0x0 // initC -v_accvgpr_write acc231, 0x0 // initC -v_accvgpr_write acc232, 0x0 // initC -v_accvgpr_write acc233, 0x0 // initC -v_accvgpr_write acc234, 0x0 // initC -v_accvgpr_write acc235, 0x0 // initC -v_accvgpr_write acc236, 0x0 // initC -v_accvgpr_write acc237, 0x0 // initC -v_accvgpr_write acc238, 0x0 // initC -v_accvgpr_write acc239, 0x0 // initC -v_accvgpr_write acc240, 0x0 // initC -v_accvgpr_write acc241, 0x0 // initC -v_accvgpr_write acc242, 0x0 // initC -v_accvgpr_write acc243, 0x0 // initC -v_accvgpr_write acc244, 0x0 // initC -v_accvgpr_write acc245, 0x0 // initC -v_accvgpr_write acc246, 0x0 // initC -v_accvgpr_write acc247, 0x0 // initC -v_accvgpr_write acc248, 0x0 // initC -v_accvgpr_write acc249, 0x0 // initC -v_accvgpr_write acc250, 0x0 // initC -v_accvgpr_write acc251, 0x0 // initC -v_accvgpr_write acc252, 0x0 // initC -v_accvgpr_write acc253, 0x0 // initC -v_accvgpr_write acc254, 0x0 // initC -v_accvgpr_write acc255, 0x0 // initC -s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? - -/* after InitC, skip to end of prefetch last iter if numIter==0 */ -s_cbranch_scc0 label_NoBranch_5L94Q2PNAOS4OEC5_0 // Only branch on scc1 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_5L94Q2PNAOS4OEC5_0: -s_waitcnt vmcnt(8) -/* local write a */ - -/* local write b */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 - -/* local write swap a */ - -/* local write swap b */ -s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // PGR=2 but only 1 loop -s_cbranch_scc1 label_skipPGR2_0 // PGR=2 but only 1 loop -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 - -label_skipPGR2_0: -s_waitcnt lgkmcnt(0) // 0prefetch wait for local write -// Skip force waitcnt0 -s_barrier - -/* local read prefetch a */ - -/* local read prefetch b */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 - -/* local read inc a */ -/* N/A, lro->32 */ -/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ - -/* local read inc b */ -/* N/A, lro->32 */ -/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 - -/******************************************/ -/* Unrolled Loop(s) - Begin */ -/******************************************/ -label_openLoopL: -s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // LoopCounterL < EndCounter -s_cbranch_scc1 label_LoopEndL_odd_NoLoadLoop -s_cmp_le_u32 s[sgprLoopCounterL], 0x2 // LoopCounterL < EndCounter -s_cbranch_scc1 label_LoopEndL_even // do not enter LoopL -label_LoopBeginL: - -/******************************************/ -/* Unrolled Loop 1/2 - Begin */ -/******************************************/ - -s_waitcnt vmcnt(8) - -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/******************************************/ -/* Unrolled Loop - End */ -/******************************************/ - -/* closeLoop loopL finalLoop=1 tailLoop=0 */ -s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL -s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==1 -s_cbranch_scc1 label_LoopEndL_odd // to End - -/******************************************/ -/* Unrolled Loop 2/2 - Begin */ -/******************************************/ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -s_waitcnt vmcnt(8) - -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 -/******************************************/ -/* Unrolled Loop - End */ -/******************************************/ - -/* closeLoop loopL finalLoop=1 tailLoop=0 */ -s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL -s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==2 -s_cbranch_scc0 label_LoopBeginL // restart LoopL -label_LoopEndL_even: - -/* Before NLL: Check VGPR.checkin for INT8 LW */ - -/******************************************/ -/* Ord. NoGlobalLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(8) -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(10) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(9) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(8) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/******************************************/ -/* Ord. NoLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(0) - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -label_Summation_End_OptNLL: -s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 -s_cbranch_scc1 label_NoBranch_PAGZHD9H2DI57HHE_0 // Only branch on scc0 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_PAGZHD9H2DI57HHE_0: - -s_cmp_eq_u32 s[sgprAlpha], 1.0 // Alpha == 1.0 ? -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_XQG82FMXJOJL8OIW_0: - -s_and_b32 s80, 255, s[sgprSizeI] // s80 = s[sgprSizeI] % 256 -s_add_u32 s81, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s81 // wg0 >= nwg0-1 ? -s_cselect_b32 s80, s80, 0 // set rMT0 -s_cmpk_gt_u32 s80, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_XQ75AI1RJ5F179IN_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_XQ75AI1RJ5F179IN_0: -s_and_b32 s80, 255, s[sgprSizeJ] // s80 = s[sgprSizeJ] % 256 -s_add_u32 s81, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s81 // wg1 >= nwg1-1 -s_cselect_b32 s80, s80, 0 // set rMT1 -s_cmpk_gt_u32 s80, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_GFF6GDU2NIUMLQ8E_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_GFF6GDU2NIUMLQ8E_0: - -s_and_b32 s81, 63, s[sgprSizesSum+0] // s81 = s[sgprSizesSum+0] % 64 -s_cmp_eq_u32 s81, 0x0 // numIterL == 0 -s_cbranch_scc1 label_NoBranch_VEIX0X0UO5PRLIZN_0 // Only branch on scc0 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_VEIX0X0UO5PRLIZN_0: - -/* endSummation: add vgpr [0...230) to pool */ -/* load store sgprs */ -.set sgprAddressScaleAlphaVec, 48 -.set sgprAddressBias, 50 -.set sgprBiasType, 52 -.set sgprBiasStride, 53 -.set sgpractivationAlpha, 54 -.set sgpractivationBeta, 55 -.set sgprActivationType, 56 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalEpilogueStruct // branch if ArgType == 2 -s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 -s_branch label_LoadExternalEpilogueStructEnd -label_LoadExternalEpilogueStruct: -s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 -s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 -s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 -label_LoadExternalEpilogueStructEnd: -.set sgprSrdScaleAlphaVec, 32 -.set sgprSrdBias, 40 - -/* Mapping of Acc register -> C Vgpr register */ -/* computeStoreVgprs */ -v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 -v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 -v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 -v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 -v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 -v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output -v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) -v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 -v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 -v_and_b32 v0, 3, v4 // v0 = v4 % 4 -v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 -v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 -v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 -s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 -v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 - -/******************************************/ -/* Global Write Elements */ -/******************************************/ -s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. -s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? -s_cbranch_scc0 label_ScaleAlphaVecAddrValid // branch if s[AddressScaleAlphaVec] != 0 -s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 -s_branch label_ScaleAlphaVecAddrValid_End -label_ScaleAlphaVecAddrValid: -s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] -label_ScaleAlphaVecAddrValid_End: - -s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE -s_add_u32 s8, s[sgprWorkGroup2], 0x1 -s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) -s_cmp_eq_u32 s8, 0x0 // bias stride = 0? -s_cselect_b32 s8, s[sgprSizeI], s8 -s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? -s_cbranch_scc0 label_BiasAddrValid // branch if s[AddressBias] != 0 -s_mov_b32 s[sgprSrdBias+2], 0 -s_branch label_BiasAddrValid_End -label_BiasAddrValid: -s_mov_b32 s[sgprSrdBias+2], s8 -label_BiasAddrValid_End: - -label_Load_Biasf32_0: -s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 -s_cbranch_scc1 label_Load_Biasbf16_0 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE -buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End // Branch to load bias end -label_Load_Biasbf16_0: -s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 -s_cbranch_scc1 label_Load_Bias_End // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE -buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End // Branch to load bias end -label_Load_Bias_End: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4 // Branch if true -label_To_Activation_None_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Abs_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Clippedrelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Gelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Leakyrelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Relu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Sigmoid_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Tanh_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Geluscaling_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Silu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_ActivationSetPCAddrEnd: -label_GW_B0_E0: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End // jump to end -label_GW_End: - -s_endpgm // Kernel End - -label_LoopEndL_odd: -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/* Before NLL: Check VGPR.checkin for INT8 LW */ - -/******************************************/ -/* Ord. NoGlobalLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(8) -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(10) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(9) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(8) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 - -s_branch label_LoopEndL_odd_NoLoadLoop - -label_LoopEndL_odd_NoLoadLoop: - -/******************************************/ -/* Ord. NoLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(0) - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -s_branch label_Summation_End_OptNLL - -label_PrefetchGlobalLastIterEnd: -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/******************************************/ -/* Tail Loop */ -/******************************************/ - -/* Tail: add ValuA/B vgpr buffer [0...160) to pool */ - -/* local write reset offsets a */ - -/* local write reset offsets b */ - -// numIterL = LOCAL_SPLITU * min(sizeL % LOCAL_DEPTHU, DEPTHU / LOCAL_SPLITU) -s_and_b32 s[sgprLoopCounterL], 63, s[sgprSizesSum+0] // s[sgprLoopCounterL] = s[sgprSizesSum+0] % 64 -s_cmp_lg_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx == numIterPerWgRemainder -s_cmov_b32 s[sgprLoopCounterL], 0x0 // numIter=0 if gsuSimIdx!=remainder -s_cmp_eq_u32 s[sgprLoopCounterL], 0x0 // numIterL == 0 -s_mov_b32 s[sgprOrigLoopCounter], 0 // repurpose to count each localRead increment -s_cbranch_scc1 label_SkipTailLoopL // skip to end of tail loop b/c numIter==0 - -/* remove stagger offsets for tail loop */ -s_sub_i32 s80, 3, s[sgprStaggerUIter] -s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes -s_mul_i32 s80, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes -s_sub_u32 s80, s80, s[sgprWrapUA] // S - WrapU -s_subb_u32 s81, s81, s[sgprWrapUA+1] // S - WrapU -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -s_sub_i32 s80, 3, s[sgprStaggerUIter] -s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes -s_mul_i32 s80, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes -s_sub_u32 s80, s80, s[sgprWrapUB] // S - WrapU -s_subb_u32 s81, s81, s[sgprWrapUB+1] // S - WrapU -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* Recalc global read offsets */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0 15, v1 // 1. M offset: mIdx = wtid % MI_M(16) -v_mul_lo_u32 v0, s[sgprStrideA0I], v0 // 1. M offset: mOffset = mIdx * mStride(k) -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) -v_mul_lo_u32 v1, s[sgprStrideA0I], v1 // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) -v_lshlrev_b32 v1, 0x6, v1 // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) -v_add_u32 v[vgprGlobalReadOffsetA], v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -v_add_u32 v[vgprGlobalReadOffsetA] 0x8 v[vgprGlobalReadOffsetA] // add prepad for pointer shift -v_lshlrev_b32 v[vgprGlobalReadOffsetA] 0x1 v[vgprGlobalReadOffsetA] // offset *= bytes/element - -s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 16 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] - -/* Update M0 for DTLDS */ - -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+0], v[vgprValuA_X0_I0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+1], v[vgprValuA_X0_I0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+8], v[vgprValuA_X0_I0+8], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+9], v[vgprValuA_X0_I0+9], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 64 -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:64 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:66 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+0], v[vgprValuA_X2_I0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:68 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:70 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+1], v[vgprValuA_X2_I0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+8], v[vgprValuA_X2_I0+8], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+9], v[vgprValuA_X2_I0+9], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+2], v[vgprValuA_X0_I0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+3], v[vgprValuA_X0_I0+3], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 64 -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+10], v[vgprValuA_X0_I0+10], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+11], v[vgprValuA_X0_I0+11], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+4], 64 -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+2], v[vgprValuA_X2_I0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+3], v[vgprValuA_X2_I0+3], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+10], v[vgprValuA_X2_I0+10], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+11], v[vgprValuA_X2_I0+11], v0 // HasEccHalf: pack -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+4], v[vgprValuA_X0_I0+4], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+5], v[vgprValuA_X0_I0+5], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 64 -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+12], v[vgprValuA_X0_I0+12], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+13], v[vgprValuA_X0_I0+13], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+5], 64 -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+4], v[vgprValuA_X2_I0+4], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+5], v[vgprValuA_X2_I0+5], v0 // HasEccHalf: pack -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+12], v[vgprValuA_X2_I0+12], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+13], v[vgprValuA_X2_I0+13], v0 // HasEccHalf: pack -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+6], v[vgprValuA_X0_I0+6], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+7], v[vgprValuA_X0_I0+7], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 64 -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+14], v[vgprValuA_X0_I0+14], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+15], v[vgprValuA_X0_I0+15], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+6], 64 -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+6], v[vgprValuA_X2_I0+6], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+7], v[vgprValuA_X2_I0+7], v0 // HasEccHalf: pack -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+14], v[vgprValuA_X2_I0+14], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+15], v[vgprValuA_X2_I0+15], v0 // HasEccHalf: pack - -/* Update M0 for DTLDS */ - -/* global read B */ -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+0+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+0], v[vgprG2LB+0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+0+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+1], v[vgprG2LB+0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+0+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:8 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+2], v[vgprG2LB+0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:12 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+3], v[vgprG2LB+0+3], v0 // HasEccHalf: pack -/* g2l=4, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+4+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // load one buffer value -/* g2l=4, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+0], v[vgprG2LB+4+0], v0 // HasEccHalf: pack -/* g2l=4, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+4+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:4 // load one buffer value -/* g2l=4, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+1], v[vgprG2LB+4+1], v0 // HasEccHalf: pack -/* g2l=4, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+4+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:8 // load one buffer value -/* g2l=4, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+2], v[vgprG2LB+4+2], v0 // HasEccHalf: pack -/* g2l=4, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:12 // load one buffer value -/* g2l=4, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+3], v[vgprG2LB+4+3], v0 // HasEccHalf: pack -/* g2l=8, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+8+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // load one buffer value -/* g2l=8, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+0], v[vgprG2LB+8+0], v0 // HasEccHalf: pack -/* g2l=8, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+8+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:4 // load one buffer value -/* g2l=8, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+1], v[vgprG2LB+8+1], v0 // HasEccHalf: pack -/* g2l=8, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+8+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:8 // load one buffer value -/* g2l=8, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+2], v[vgprG2LB+8+2], v0 // HasEccHalf: pack -/* g2l=8, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:12 // load one buffer value -/* g2l=8, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+3], v[vgprG2LB+8+3], v0 // HasEccHalf: pack -/* g2l=12, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+12+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // load one buffer value -/* g2l=12, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+0], v[vgprG2LB+12+0], v0 // HasEccHalf: pack -/* g2l=12, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+12+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:4 // load one buffer value -/* g2l=12, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+1], v[vgprG2LB+12+1], v0 // HasEccHalf: pack -/* g2l=12, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+12+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:8 // load one buffer value -/* g2l=12, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+2], v[vgprG2LB+12+2], v0 // HasEccHalf: pack -/* g2l=12, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:12 // load one buffer value -/* g2l=12, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+3], v[vgprG2LB+12+3], v0 // HasEccHalf: pack -/* g2l=16, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+16+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // load one buffer value -/* g2l=16, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+0], v[vgprG2LB+16+0], v0 // HasEccHalf: pack -/* g2l=16, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+16+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:4 // load one buffer value -/* g2l=16, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+1], v[vgprG2LB+16+1], v0 // HasEccHalf: pack -/* g2l=16, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+16+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:8 // load one buffer value -/* g2l=16, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+2], v[vgprG2LB+16+2], v0 // HasEccHalf: pack -/* g2l=16, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:12 // load one buffer value -/* g2l=16, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+3], v[vgprG2LB+16+3], v0 // HasEccHalf: pack -/* g2l=20, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+20+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // load one buffer value -/* g2l=20, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+0], v[vgprG2LB+20+0], v0 // HasEccHalf: pack -/* g2l=20, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+20+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:4 // load one buffer value -/* g2l=20, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+1], v[vgprG2LB+20+1], v0 // HasEccHalf: pack -/* g2l=20, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+20+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:8 // load one buffer value -/* g2l=20, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+2], v[vgprG2LB+20+2], v0 // HasEccHalf: pack -/* g2l=20, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:12 // load one buffer value -/* g2l=20, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+3], v[vgprG2LB+20+3], v0 // HasEccHalf: pack -/* g2l=24, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+24+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // load one buffer value -/* g2l=24, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+0], v[vgprG2LB+24+0], v0 // HasEccHalf: pack -/* g2l=24, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+24+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:4 // load one buffer value -/* g2l=24, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+1], v[vgprG2LB+24+1], v0 // HasEccHalf: pack -/* g2l=24, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+24+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:8 // load one buffer value -/* g2l=24, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+2], v[vgprG2LB+24+2], v0 // HasEccHalf: pack -/* g2l=24, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:12 // load one buffer value -/* g2l=24, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+3], v[vgprG2LB+24+3], v0 // HasEccHalf: pack -/* g2l=28, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+28+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // load one buffer value -/* g2l=28, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+0], v[vgprG2LB+28+0], v0 // HasEccHalf: pack -/* g2l=28, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+28+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:4 // load one buffer value -/* g2l=28, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+1], v[vgprG2LB+28+1], v0 // HasEccHalf: pack -/* g2l=28, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+28+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:8 // load one buffer value -/* g2l=28, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+2], v[vgprG2LB+28+2], v0 // HasEccHalf: pack -/* g2l=28, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:12 // load one buffer value -/* g2l=28, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+3], v[vgprG2LB+28+3], v0 // HasEccHalf: pack -s_waitcnt vmcnt(0) // 2wait for global read -// Skip force waitcnt0 -s_barrier - -/* local write a */ - -/* local write b */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 - -/* Recalc local read offsets */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x2, v2 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s8, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 -/* N/A */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s8, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 -s_waitcnt lgkmcnt(0) // 5wait for local write -// Skip force waitcnt0 -s_barrier - -/* local read reset offsets a */ - -/* local read reset offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ - -/* tail loop: macs */ -label_TailLoopBeginL: - -/* Tail: remove ValuA/B vgpr buffer [0...160) from pool */ - -/* Tail: add address/G2L vgpr [160...230) to pool */ - -/* local read a */ - -/* local read b */ -ds_read_b64 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+1], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+2+1], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+1], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+6+1], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+1], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+10+1], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+1], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+14+1], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+1], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+18:vgprValuB_X0_I0+18+1], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+1], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+22:vgprValuB_X0_I0+22+1], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+1], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+26:vgprValuB_X0_I0+26+1], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+1], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+30:vgprValuB_X0_I0+30+1], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 - -/* local read inc a */ -s_mov_b32 s8, 0x20 // inc - -/* local read inc b */ -s_mov_b32 s8, 0x20 // inc -v_add_co_u32 v[vgprLocalReadAddrB], vcc, s8, v[vgprLocalReadAddrB] // lrB += 32 (bpeDS) -s_waitcnt lgkmcnt(0) // 4wait for local read -v_and_b32 v160, 63, v[vgprSerial] // v160 = v[vgprSerial] % 64 -v_lshrrev_b32 v160, 4, v160 // v160 = v160 / 16 -v_lshlrev_b32 v160, 0x2, v160 // v160 = v160 * 4 -v_cmp_ge_i32 s[80:81], v160, s[sgprLoopCounterL] // check K index >= Size L -v_cndmask_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+0+1], v[vgprValuA_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+2+1], v[vgprValuA_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+4+1], v[vgprValuA_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+6+1], v[vgprValuA_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+0+0], v[vgprValuB_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+2+0], v[vgprValuB_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+4+0], v[vgprValuB_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+6+0], v[vgprValuB_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+8+0], v[vgprValuB_X0_I0+8+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+10+0], v[vgprValuB_X0_I0+10+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+12+0], v[vgprValuB_X0_I0+12+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+14+0], v[vgprValuB_X0_I0+14+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+16+0], v[vgprValuB_X0_I0+16+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+18+0], v[vgprValuB_X0_I0+18+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+20+0], v[vgprValuB_X0_I0+20+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+22+0], v[vgprValuB_X0_I0+22+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+24+0], v[vgprValuB_X0_I0+24+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+26+0], v[vgprValuB_X0_I0+26+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+28+0], v[vgprValuB_X0_I0+28+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+30+0], v[vgprValuB_X0_I0+30+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+0+1], v[vgprValuB_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+2+1], v[vgprValuB_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+4+1], v[vgprValuB_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+6+1], v[vgprValuB_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+8+1], v[vgprValuB_X0_I0+8+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+10+1], v[vgprValuB_X0_I0+10+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+12+1], v[vgprValuB_X0_I0+12+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+14+1], v[vgprValuB_X0_I0+14+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+16+1], v[vgprValuB_X0_I0+16+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+18+1], v[vgprValuB_X0_I0+18+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+20+1], v[vgprValuB_X0_I0+20+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+22+1], v[vgprValuB_X0_I0+22+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+24+1], v[vgprValuB_X0_I0+24+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+26+1], v[vgprValuB_X0_I0+26+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+28+1], v[vgprValuB_X0_I0+28+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+30+1], v[vgprValuB_X0_I0+30+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_sub_u32 v160, s[sgprLoopCounterL], v160 // get distance between size and k index -v_cmp_lt_i32 s[80:81], v160, 4 // set partial 0 if distance less than input per thread -s_and_b32 s82, s[sgprLoopCounterL], 3 // get inputs for edge thread -s_sub_u32 s82, 4, s82 // use shift to fill 0 for outside element -s_lshl_b32 s82, s82, 4 // use shift to fill 0 for outside element -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+0], v[vgprValuA_X0_I0+0+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+0], v[vgprValuA_X0_I0+2+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+0], v[vgprValuA_X0_I0+4+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+0], v[vgprValuA_X0_I0+6+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+0], v[vgprValuB_X0_I0+0+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+1], v[vgprValuB_X0_I0+0+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+0], v[vgprValuB_X0_I0+2+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+1], v[vgprValuB_X0_I0+2+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+0], v[vgprValuB_X0_I0+4+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+1], v[vgprValuB_X0_I0+4+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+0], v[vgprValuB_X0_I0+6+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+1], v[vgprValuB_X0_I0+6+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+0], v[vgprValuB_X0_I0+8+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+1], v[vgprValuB_X0_I0+8+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+0], v[vgprValuB_X0_I0+10+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+1], v[vgprValuB_X0_I0+10+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+0], v[vgprValuB_X0_I0+12+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+1], v[vgprValuB_X0_I0+12+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+0], v[vgprValuB_X0_I0+14+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+1], v[vgprValuB_X0_I0+14+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+0], v[vgprValuB_X0_I0+16+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+1], v[vgprValuB_X0_I0+16+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+0], v[vgprValuB_X0_I0+18+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+1], v[vgprValuB_X0_I0+18+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+0], v[vgprValuB_X0_I0+20+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+1], v[vgprValuB_X0_I0+20+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+0], v[vgprValuB_X0_I0+22+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+1], v[vgprValuB_X0_I0+22+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+0], v[vgprValuB_X0_I0+24+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+1], v[vgprValuB_X0_I0+24+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+0], v[vgprValuB_X0_I0+26+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+1], v[vgprValuB_X0_I0+26+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+0], v[vgprValuB_X0_I0+28+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+1], v[vgprValuB_X0_I0+28+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+0], v[vgprValuB_X0_I0+30+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+1], v[vgprValuB_X0_I0+30+0+0+1], v163, s[80:81] -s_nop 1 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[252:255] // left value = acc[252+0:255+0] - -v_mov_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+8+0] -v_mov_b32 v[vgprValuA_X0_I0+1+0], v[vgprValuA_X0_I0+9+0] -v_mov_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+10+0] -v_mov_b32 v[vgprValuA_X0_I0+3+0], v[vgprValuA_X0_I0+11+0] -v_mov_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+12+0] -v_mov_b32 v[vgprValuA_X0_I0+5+0], v[vgprValuA_X0_I0+13+0] -v_mov_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+14+0] -v_mov_b32 v[vgprValuA_X0_I0+7+0], v[vgprValuA_X0_I0+15+0] -v_mov_b32 v[vgprValuA_X0_I0+8+0], v[vgprValuA_X2_I0+0+0] -v_mov_b32 v[vgprValuA_X0_I0+9+0], v[vgprValuA_X2_I0+1+0] -v_mov_b32 v[vgprValuA_X0_I0+10+0], v[vgprValuA_X2_I0+2+0] -v_mov_b32 v[vgprValuA_X0_I0+11+0], v[vgprValuA_X2_I0+3+0] -v_mov_b32 v[vgprValuA_X0_I0+12+0], v[vgprValuA_X2_I0+4+0] -v_mov_b32 v[vgprValuA_X0_I0+13+0], v[vgprValuA_X2_I0+5+0] -v_mov_b32 v[vgprValuA_X0_I0+14+0], v[vgprValuA_X2_I0+6+0] -v_mov_b32 v[vgprValuA_X0_I0+15+0], v[vgprValuA_X2_I0+7+0] -v_mov_b32 v[vgprValuA_X2_I0+0+0], v[vgprValuA_X2_I0+8+0] -v_mov_b32 v[vgprValuA_X2_I0+1+0], v[vgprValuA_X2_I0+9+0] -v_mov_b32 v[vgprValuA_X2_I0+2+0], v[vgprValuA_X2_I0+10+0] -v_mov_b32 v[vgprValuA_X2_I0+3+0], v[vgprValuA_X2_I0+11+0] -v_mov_b32 v[vgprValuA_X2_I0+4+0], v[vgprValuA_X2_I0+12+0] -v_mov_b32 v[vgprValuA_X2_I0+5+0], v[vgprValuA_X2_I0+13+0] -v_mov_b32 v[vgprValuA_X2_I0+6+0], v[vgprValuA_X2_I0+14+0] -v_mov_b32 v[vgprValuA_X2_I0+7+0], v[vgprValuA_X2_I0+15+0] - -/* closeLoop loopL finalLoop=1 tailLoop=1 */ -s_sub_i32 s[sgprLoopCounterL], s[sgprLoopCounterL], 0x10 // dec counterL (tailLoop) -s_add_u32 s[sgprOrigLoopCounter], s[sgprOrigLoopCounter], 0x10 // inc counterL -s_cmp_le_i32 s[sgprLoopCounterL], 0x0 // counterL<=0 -s_cbranch_scc0 label_TailLoopBeginL // restart LoopL -label_TailLoopEndL: -label_SkipTailLoopL: - -/* Tail: remove address/G2L [160...230) from pool */ -label_Summation_End_SB904UR36QKZ73J2_0: -/* endSummation: add vgpr [0...230) to pool */ -.set sgprWGM, UNDEF -.set sgprLoopCounterL, UNDEF -.set sgprOrigLoopCounter, UNDEF -.set sgprAddressA, UNDEF -.set sgprAddressB, UNDEF -.set sgprStridesA, UNDEF -.set sgprStridesB, UNDEF -.set sgprStaggerUIter, UNDEF -.set sgprSrdA, UNDEF -.set sgprSrdB, UNDEF -.set sgprShadowLimitA, UNDEF -.set sgprShadowLimitB, UNDEF -.set sgprWrapUA, UNDEF -.set sgprWrapUB, UNDEF -.set sgprGlobalReadIncsA, UNDEF -.set sgprGlobalReadIncsB, UNDEF -.set sgprScalarGlobalReadOffsetA, UNDEF -.set sgprScalarGlobalReadOffsetB, UNDEF -/* load store sgprs */ -.set sgprAddressScaleAlphaVec, 48 -.set sgprAddressBias, 50 -.set sgprBiasType, 52 -.set sgprBiasStride, 53 -.set sgpractivationAlpha, 54 -.set sgpractivationBeta, 55 -.set sgprActivationType, 56 -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc0 label_GSU_4 // branch if GSU != 1 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalEpilogueStruct_1 // branch if ArgType == 2 -s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 -s_branch label_LoadExternalEpilogueStructEnd_1 -label_LoadExternalEpilogueStruct_1: -s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 -s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 -s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 -label_LoadExternalEpilogueStructEnd_1: -label_GSU_4: -.set sgprSrdScaleAlphaVec, 32 -.set sgprSrdBias, 40 - -/* Mapping of Acc register -> C Vgpr register */ - -/* not-LocalSplitU: global write indices */ -/* computeStoreVgprs */ -v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 -v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 -v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 -v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 -v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 -v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output -v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) -v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 -v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 -v_and_b32 v0, 3, v4 // v0 = v4 % 4 -v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 -v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 -v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 -s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 -v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 - -/* not-LocalSplitU: global write */ - -/******************************************/ -/* Global Write Elements */ -/******************************************/ -s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_5 // branch if GSU == 1 -s_and_b32 s58, 255, s[sgprSizeI] // s58 = s[sgprSizeI] % 256 -s_add_u32 s59, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s59 // wg0 >= nwg0-1 ? -s_cselect_b32 s58, s58, 0 // set rMT0 -s_cmpk_gt_u32 s58, 0x0 // rMT0 > 0 -s_cbranch_scc1 label_GW_B0_E1_M // jump if edges required -s_and_b32 s58, 255, s[sgprSizeJ] // s58 = s[sgprSizeJ] % 256 -s_add_u32 s59, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s59 // wg1 >= nwg1-1 -s_cselect_b32 s58, s58, 0 // set rMT1 -s_cmpk_gt_u32 s58, 0x0 // rMT1 > 0 -s_cbranch_scc1 label_GW_B0_E1_N // jump if edges required -label_GW_B0_E0_1: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v10, v3, v0, 0x2 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+20], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+21], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+22], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+23], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+24], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+25], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+26], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+27], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+28], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+29], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+31], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+32], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+33], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+34], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+35], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+36], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+37], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+38], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+39], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+40], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+41], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+42], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+43], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+44], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+45], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+46], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+47], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+48], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+49], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+50], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+51], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+52], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+53], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+54], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+55], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+56], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+57], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+58], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+59], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+60], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+61], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+62], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+63], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+64], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+65], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+66], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+67], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+68], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+69], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+70], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+71], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+73], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+74], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+75], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+20], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+21], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+22], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+23], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+24], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+25], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+26], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+27], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+28], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+29], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+31], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+32], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+33], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+34], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+35], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+36], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+37], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+38], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+39], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+40], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+41], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+42], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+43], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+44], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+45], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+46], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+47], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+48], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+49], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+50], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+51], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+52], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+53], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+54], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+55], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+60], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+61], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+62], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+63], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+64], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+65], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+66], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+67], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+68], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+69], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+70], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+71], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+73], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+74], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+75], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+20], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+21], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+22], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+23], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+24], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+25], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+26], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+27], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+32], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+33], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+34], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+35], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+36], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+37], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+38], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+39], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+40], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+41], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+42], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+43], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+44], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+45], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+46], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+47], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+48], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+49], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+50], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+51], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+52], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+53], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+54], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+55], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+56], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+57], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+58], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+59], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+60], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+61], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+62], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+63], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+64], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+65], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+66], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+67], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+68], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+69], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+70], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+71], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+73], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+74], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+75], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+20], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+21], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+22], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+23], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+24], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+25], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+26], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+27], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+28], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+29], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+31], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+32], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+33], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+34], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+35], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+36], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+37], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+38], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+39], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+40], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+41], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+42], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+43], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+44], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+45], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+46], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+47], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+48], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+49], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+50], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+51], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+52], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+53], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+54], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+55], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+56], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+57], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+58], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+59], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+60], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+61], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+62], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+63], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+64], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+65], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+66], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+67], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+73], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+74], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+75], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_B0_E1_N: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+24], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+25], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+26], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+27], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+28], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+29], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+30], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+31], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+32], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+33], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+34], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+35], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+36], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+37], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+38], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+39], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+44], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+45], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+46], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+47], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+48], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+49], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+50], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+51], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+52], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+53], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+54], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+55], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+56], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+57], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+58], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+59], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+64], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+65], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+66], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+67], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+68], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+69], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+70], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+71], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+72], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+73], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+74], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+75], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+76], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+77], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+78], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+79], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+24], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+25], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+26], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+27], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+28], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+29], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+30], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+31], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+32], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+33], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+34], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+35], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+36], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+37], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+38], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+39], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+44], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+45], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+46], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+47], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+48], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+49], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+50], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+51], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+52], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+53], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+54], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+55], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+56], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+57], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+58], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+59], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+64], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+65], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+66], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+67], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+68], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+69], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+70], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+71], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+72], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+73], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+74], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+75], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+76], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+77], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+78], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+79], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+24], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+25], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+26], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+27], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+28], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+29], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+30], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+31], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+32], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+33], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+34], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+35], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+48], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+49], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+50], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+51], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+52], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+53], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+54], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+55], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+56], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+57], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+58], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+59], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+64], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+65], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+66], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+67], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+68], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+69], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+70], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+71], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+72], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+73], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+74], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+75], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+76], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+77], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+78], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+79], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+24], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+25], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+26], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+27], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+28], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+29], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+30], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+31], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+32], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+33], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+34], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+35], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+36], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+37], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+38], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+39], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+44], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+45], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+46], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+47], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+48], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+49], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+50], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+51], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+52], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+53], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+54], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+55], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+56], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+57], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+58], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+59], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+64], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+65], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+66], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+67], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+68], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+69], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+70], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+71], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+72], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+73], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+74], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+75], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+76], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+77], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+78], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+79], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_B0_E1_M: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+15], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+17], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+19], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+21], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+23], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+25], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+27], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+29], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+31], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+33], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+35], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+37], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+39], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+41], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+13], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+15], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+17], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+19], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+21], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+23], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+25], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+27], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+29], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+31], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+33], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+35], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+37], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+39], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+41], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+13], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+15], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+17], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+19], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+21], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+23], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+25], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+27], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+29], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+31], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+33], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+35], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+37], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+39], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+41], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+13], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+15], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+17], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+19], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+21], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+23], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+25], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+27], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+29], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+31], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+33], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+35], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+37], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+39], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+41], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+15], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+17], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+19], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+21], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+23], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+25], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+27], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+29], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+31], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+33], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+35], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+37], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+39], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+41], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+13], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+15], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+17], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+19], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+21], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+23], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+25], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+27], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+29], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+31], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+33], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+35], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+37], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+39], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+41], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+13], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+15], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+17], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+19], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+21], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+23], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+25], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+27], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+29], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+31], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+33], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+35], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+37], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+39], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+41], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+13], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+15], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+17], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+19], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+21], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+23], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+25], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+27], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+29], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+31], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+33], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+35], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+37], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+39], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+41], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+15], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+17], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+19], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+21], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+23], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+25], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+27], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+29], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+31], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+33], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+35], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+37], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+39], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+41], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+13], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+15], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+17], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+19], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+21], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+23], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+25], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+27], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+29], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+31], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+33], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+35], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+37], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+39], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+41], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+13], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+15], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+17], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+19], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+21], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+23], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+25], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+27], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+29], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+31], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+33], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+35], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+37], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+39], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+41], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+13], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+15], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+17], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+19], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+21], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+23], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+25], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+27], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+29], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+31], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+33], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+35], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+37], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+39], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+41], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+15], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+17], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+19], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+21], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+23], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+25], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+27], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+29], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+31], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+33], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+35], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+37], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+39], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+41], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+13], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+15], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+17], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+19], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+21], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+23], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+25], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+27], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+29], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+31], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+33], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+35], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+37], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+39], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+41], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+13], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+15], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+17], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+19], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+21], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+23], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+25], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+27], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+29], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+31], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+33], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+35], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+37], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+39], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+41], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+13], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+15], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+17], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+19], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+21], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+23], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+25], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+27], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+29], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+31], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+33], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+35], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+37], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+39], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+41], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_End_1: -s_getpc_b64 s[58:59] // addr of next instr -s_add_i32 s60, label_KernelEnd, 0x4 // target branch offset -s_add_u32 s58, s58, s60 // add target branch offset -s_addc_u32 s59, s59, 0 // add high and carry -s_setpc_b64 s[58:59] // branch to label_KernelEnd -label_GSU_5: -s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? -s_cbranch_scc0 label_ScaleAlphaVec_1AddrValid // branch if s[AddressScaleAlphaVec] != 0 -s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 -s_branch label_ScaleAlphaVec_1AddrValid_End -label_ScaleAlphaVec_1AddrValid: -s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] -label_ScaleAlphaVec_1AddrValid_End: - -s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE -s_add_u32 s8, s[sgprWorkGroup2], 0x1 -s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) -s_cmp_eq_u32 s8, 0x0 // bias stride = 0? -s_cselect_b32 s8, s[sgprSizeI], s8 -s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? -s_cbranch_scc0 label_Bias_1AddrValid // branch if s[AddressBias] != 0 -s_mov_b32 s[sgprSrdBias+2], 0 -s_branch label_Bias_1AddrValid_End -label_Bias_1AddrValid: -s_mov_b32 s[sgprSrdBias+2], s8 -label_Bias_1AddrValid_End: - -label_Load_Biasf32_0_1: -s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 -s_cbranch_scc1 label_Load_Biasbf16_0_1 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE -buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End_1 // Branch to load bias end -label_Load_Biasbf16_0_1: -s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 -s_cbranch_scc1 label_Load_Bias_End_1 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE -buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End_1 // Branch to load bias end -label_Load_Bias_End_1: -s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 -s_cbranch_scc0 label_GW_Beta_2 // Branch if Beta is not zero - -s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? -s_cselect_b32 s60, s60, 0 // set rMT0 -s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_PFO42GJLMDBXSWVP_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B0_E1_M_1, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_M_1 -label_NoBranch_PFO42GJLMDBXSWVP_0: -s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 -s_cselect_b32 s60, s60, 0 // set rMT1 -s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_XR5UBTMCR0HWLF5H_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B0_E1_N_1, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_N_1 -label_NoBranch_XR5UBTMCR0HWLF5H_0: -label_GW_B0_E0_2: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_0 // Branch if true -label_To_Activation_None_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Abs_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Gelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Relu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Tanh_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Silu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_ActivationSetPCAddrEnd_6: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_B0_E1_N_1: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_1 // Branch if true -label_To_Activation_None_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Abs_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Gelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Relu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Tanh_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Silu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_ActivationSetPCAddrEnd_5: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=14 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+25], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+26], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+27], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+68], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+69], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+70], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+71], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+76], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+77], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+78], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+79], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+80], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+81], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+82], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+83], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+88], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+89], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+90], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+91], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+96], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+97], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+98], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+99], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+104], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+105], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+106], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+107], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+108], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+109], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+110], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+111], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+116], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+117], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+118], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+119], acc61 // copy acc to vreg[55] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+25], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+26], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+27], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+32], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+33], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+34], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+35], acc63 // copy acc to vreg[63] -v_accvgpr_read_b32 v[vgprValuC+40], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+41], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+42], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+43], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+48], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+49], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+50], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+51], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+52], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+53], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+54], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+55], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+60], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+61], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+62], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+63], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+68], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+69], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+70], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+71], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+76], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+77], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+78], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+79], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+80], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+81], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+82], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+83], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+88], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+89], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+90], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+91], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+96], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+97], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+98], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+99], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+104], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+105], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+106], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+107], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+108], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+109], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+110], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+111], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+116], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+117], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+118], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+119], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+25], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+26], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+27], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+32], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+33], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+34], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+35], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+40], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+41], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+42], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+43], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+48], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+49], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+50], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+51], acc127 // copy acc to vreg[127] -v_accvgpr_read_b32 v[vgprValuC+52], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+53], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+54], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+55], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+60], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+61], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+62], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+63], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+68], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+69], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+70], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+71], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+76], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+77], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+78], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+79], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+80], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+81], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+82], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+83], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+88], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+89], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+90], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+91], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+96], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+97], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+98], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+99], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+104], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+105], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+106], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+107], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+108], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+109], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+110], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+111], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+116], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+117], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+118], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+119], acc173 // copy acc to vreg[167] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4); (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+25], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+26], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+27], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+32], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+33], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+34], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+35], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+40], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+41], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+42], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+43], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+48], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+49], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+50], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+51], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+52], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+53], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+54], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+55], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+60], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+61], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+62], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+63], acc191 // copy acc to vreg[191] -v_accvgpr_read_b32 v[vgprValuC+68], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+69], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+70], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+71], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+76], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+77], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+78], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+79], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+80], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+81], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+82], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+83], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+88], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+89], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+90], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+91], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+96], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+97], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+98], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+99], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+104], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+105], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+106], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+107], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+108], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+109], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+110], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+111], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+116], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+117], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+118], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+119], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0), (12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v73, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v73, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v73, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v73, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v73, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v73, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v73, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v73, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v73, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v73, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v73, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v73, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v73, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v73, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v73, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v73, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v73, v66, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+25], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+26], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+27], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+32], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+33], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+34], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+35], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+40], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+41], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+42], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+43], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+48], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+49], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+50], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+51], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+52], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+53], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+54], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+55], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+60], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+61], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+62], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+63], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+76], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+77], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+78], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+79], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_B0_E1_M_1: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_0_edge_1 // Branch if true -label_To_Activation_None_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Abs_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Clippedrelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Gelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Leakyrelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Relu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Sigmoid_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Tanh_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Geluscaling_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Silu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_ActivationSetPCAddrEnd_4: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+24], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+36], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+40], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+44], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+48], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+52], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+56], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+60], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+64], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+68], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+72], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+76], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+80], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+84], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+24], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+36], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+40], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+44], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+48], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+52], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+56], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+60], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+64], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+68], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+76], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+80], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+84], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+24], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+30], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+36], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+40], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+44], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+48], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+52], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+56], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+60], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+64], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+68], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+76], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+80], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+84], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+24], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+36], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+40], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+44], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+48], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+52], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+56], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+60], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+64], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+68], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+76], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+80], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+84], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+24], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+36], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+40], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+44], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+48], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+52], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+56], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+60], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+64], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+68], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+72], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+76], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+80], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+84], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+24], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+36], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+40], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+44], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+48], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+52], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+56], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+60], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+64], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+68], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+76], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+80], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+84], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+24], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+36], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+40], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+44], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+48], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+52], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+56], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+60], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+64], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+68], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+76], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+80], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+84], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+24], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+30], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+36], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+40], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+44], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+48], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+52], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+56], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+60], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+64], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+68], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+76], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+80], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+84], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+24], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+36], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+40], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+44], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+48], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+52], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+56], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+60], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+64], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+68], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+72], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+76], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+80], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+84], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+24], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+36], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+40], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+44], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+48], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+52], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+56], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+60], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+64], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+68], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+76], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+80], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+84], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+24], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+30], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+36], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+40], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+44], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+48], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+52], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+56], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+60], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+64], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+68], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+76], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+80], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+84], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+24], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+30], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+36], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+40], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+44], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+48], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+52], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+56], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+60], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+64], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+68], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+76], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+80], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+84], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+24], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+36], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+40], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+44], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+48], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+52], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+56], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+60], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+64], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+68], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+72], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+76], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+80], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+84], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+24], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+36], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+40], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+44], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+48], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+52], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+56], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+60], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+64], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+68], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+76], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+80], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+84], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+24], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+30], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+36], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+40], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+44], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+48], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+52], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+56], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+60], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+64], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+68], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+76], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+80], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+84], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+24], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+36], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+40], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+44], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+48], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+52], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+56], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+60], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+64], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+68], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+76], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+80], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+84], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_Beta_2: -s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? -s_cselect_b32 s60, s60, 0 // set rMT0 -s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_1L38YJQL3BUJ48XK_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B1_E1_M, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_M -label_NoBranch_1L38YJQL3BUJ48XK_0: -s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 -s_cselect_b32 s60, s60, 0 // set rMT1 -s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_XMVL70A9XU3BIJFQ_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B1_E1_N, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_N -label_NoBranch_XMVL70A9XU3BIJFQ_0: -label_GW_B1_E0: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_0 // Branch if true -label_To_Activation_None_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Abs_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Gelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Relu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Tanh_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Silu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_ActivationSetPCAddrEnd_3: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_add_lshl_u32 v14, v2, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+64], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+65], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+66], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+67], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+73], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+74], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+75], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+76], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+77], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+78], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+79], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+84], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+85], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+86], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+87], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+88], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+89], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+90], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+91], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+96], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+97], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+98], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+99], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+100], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+101], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+102], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+103], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+108], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+109], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+110], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+111], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+112], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+113], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+114], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+115], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+120], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+121], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+122], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+123], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+36], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+37], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+38], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+39], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+40], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+41], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+42], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+43], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+48], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+49], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+50], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+51], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+52], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+53], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+54], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+55], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+60], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+61], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+62], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+63], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+64], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+65], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+66], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+67], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+73], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+74], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+75], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+76], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+77], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+78], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+79], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+84], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+85], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+86], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+87], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+88], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+89], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+90], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+91], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+96], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+97], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+98], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+99], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+100], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+101], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+102], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+103], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+108], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+109], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+110], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+111], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+112], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+113], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+114], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+115], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+120], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+121], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+122], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+123], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+36], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+37], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+38], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+39], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+40], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+41], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+42], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+43], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+48], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+49], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+50], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+51], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+52], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+53], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+54], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+55], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+60], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+61], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+62], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+63], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+64], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+65], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+66], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+67], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+73], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+74], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+75], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+76], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+77], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+78], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+79], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+84], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+85], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+86], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+87], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+88], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+89], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+90], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+91], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+96], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+97], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+98], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+99], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+112], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+113], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+114], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+115], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+120], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+121], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+122], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+123], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+40], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+41], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+42], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+43], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+48], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+49], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+50], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+51], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+52], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+53], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+54], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+55], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+60], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+61], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+62], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+63], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+64], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+65], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+66], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+67], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+73], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+74], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+75], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+76], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+77], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+78], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+79], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+84], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+85], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+86], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+87], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+88], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+89], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+90], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+91], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+96], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+97], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+98], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+99], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+100], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+101], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+102], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+103], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+108], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+109], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+110], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+111], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+112], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+113], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+114], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+115], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+120], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+121], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+122], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+123], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_GW_B1_E1_N: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_1 // Branch if true -label_To_Activation_None_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Abs_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Gelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Relu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Tanh_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Silu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_ActivationSetPCAddrEnd_2: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=12 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+44], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+45], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+46], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+47], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+56], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+57], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+58], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+59], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+64], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+65], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+66], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+67], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+72], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+73], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+74], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+75], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+80], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+81], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+82], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+83], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+92], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+93], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+94], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+95], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+100], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+101], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+102], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+103], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+108], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+109], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+110], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+111], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+116], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+117], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+118], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+119], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+128], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+129], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+130], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+131], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+29], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+31], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+36], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+37], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+38], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+39], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+44], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+45], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+46], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+47], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+56], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+57], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+58], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+59], acc63 // copy acc to vreg[63] -v_accvgpr_read_b32 v[vgprValuC+64], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+65], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+66], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+67], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+72], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+73], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+74], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+75], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+80], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+81], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+82], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+83], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+92], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+93], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+94], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+95], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+100], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+101], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+102], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+103], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+108], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+109], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+110], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+111], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+116], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+117], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+118], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+119], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+128], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+129], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+130], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+131], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+29], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+31], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+36], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+37], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+38], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+39], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+44], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+45], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+46], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+47], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+64], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+65], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+66], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+67], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+72], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+73], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+74], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+75], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+80], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+81], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+82], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+83], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+92], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+93], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+94], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+95], acc127 // copy acc to vreg[127] -v_accvgpr_read_b32 v[vgprValuC+100], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+101], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+102], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+103], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+108], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+109], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+110], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+111], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+116], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+117], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+118], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+119], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+128], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+129], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+130], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+131], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+64], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+65], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+66], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+67], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+72], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+73], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+74], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+75], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+80], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+81], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+82], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+83], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+92], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+93], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+94], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+95], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+116], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+117], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+118], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+119], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+128], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+129], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+130], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+131], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+44], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+45], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+46], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+47], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+56], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+57], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+58], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+59], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+64], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+65], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+66], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+67], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+72], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+73], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+74], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+75], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+80], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+81], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+82], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+83], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+92], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+93], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+94], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+95], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+100], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+101], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+102], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+103], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+108], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+109], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+110], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+111], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+116], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+117], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+118], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+119], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+128], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+129], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+130], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+131], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v51, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v51, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v51, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v51, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v51, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v51, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v51, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v51, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v51, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v51, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v51, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v51, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v51, v48, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+29], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+31], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+36], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+37], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+38], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+39], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+44], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+45], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+46], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+47], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+56], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+57], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+58], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+59], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_GW_B1_E1_M: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_1_edge_1 // Branch if true -label_To_Activation_None_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Abs_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Clippedrelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Gelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Leakyrelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Relu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Sigmoid_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Tanh_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Geluscaling_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Silu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_ActivationSetPCAddrEnd_1: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+26], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+33], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+40], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+45], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+50], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+55], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+60], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+65], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+70], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+75], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+80], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+85], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+90], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+95], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+100], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+26], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+33], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+40], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+45], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+50], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+55], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+60], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+65], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+70], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+75], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+80], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+85], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+90], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+95], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+100], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+26], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+33], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+40], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+45], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+50], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+55], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+60], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+65], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+70], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+75], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+80], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+85], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+90], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+95], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+100], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+26], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+33], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+40], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+45], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+50], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+55], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+60], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+65], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+70], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+75], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+80], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+85], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+90], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+95], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+100], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+26], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+33], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+40], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+45], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+50], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+55], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+60], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+65], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+70], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+75], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+80], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+85], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+90], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+95], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+100], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+26], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+33], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+40], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+45], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+50], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+55], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+60], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+65], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+70], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+75], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+80], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+85], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+90], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+95], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+100], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+26], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+33], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+40], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+45], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+50], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+55], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+60], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+65], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+70], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+75], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+80], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+85], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+90], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+95], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+100], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+26], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+33], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+40], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+45], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+50], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+55], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+60], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+65], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+70], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+75], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+80], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+85], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+90], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+95], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+100], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+26], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+33], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+40], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+45], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+50], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+55], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+60], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+65], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+70], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+75], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+80], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+85], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+90], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+95], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+100], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+26], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+33], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+40], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+45], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+50], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+55], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+60], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+65], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+70], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+75], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+80], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+85], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+90], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+95], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+100], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+26], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+33], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+40], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+45], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+50], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+55], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+60], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+65], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+70], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+75], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+80], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+85], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+90], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+95], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+100], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+26], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+33], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+40], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+45], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+50], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+55], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+60], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+65], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+70], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+75], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+80], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+85], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+90], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+95], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+100], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+26], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+33], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+40], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+45], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+50], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+55], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+60], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+65], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+70], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+75], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+80], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+85], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+90], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+95], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+100], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+26], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+33], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+40], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+45], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+50], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+55], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+60], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+65], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+70], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+75], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+80], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+85], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+90], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+95], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+100], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+26], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+33], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+40], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+45], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+50], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+55], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+60], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+65], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+70], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+75], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+80], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+85], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+90], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+95], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+100], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+26], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+33], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+40], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+45], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+50], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+55], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+60], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+65], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+70], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+75], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+80], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+85], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+90], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+95], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+100], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_Activation_None_VW1: -s_setpc_b64 s[58:59] -label_Activation_Abs_VW1: -v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit -s_setpc_b64 s[58:59] -label_Activation_Clippedrelu_VW1: -v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) -v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha -s_setpc_b64 s[58:59] -label_Activation_Gelu_VW1: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -s_setpc_b64 s[58:59] -label_Activation_Leakyrelu_VW1: -v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha -v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? -v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 -s_setpc_b64 s[58:59] -label_Activation_Relu_VW1: -v_max_f32 v4, v4, 0 // x = max(0, x) -s_setpc_b64 s[58:59] -label_Activation_Sigmoid_VW1: -v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // 1 + exp(-x) -v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -s_setpc_b64 s[58:59] -label_Activation_Tanh_VW1: -v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha -v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // e^2x + 1 -v_rcp_f32 v4, v4 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) -s_setpc_b64 s[58:59] -label_Activation_Geluscaling_VW1: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -s_setpc_b64 s[58:59] -label_Activation_Silu_VW1: -v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) -s_setpc_b64 s[58:59] -label_GW_End_2: -label_KernelEnd: -s_endpgm // Kernel End -label_Activation_None_VW4: -s_setpc_b64 s[58:59] -label_Activation_Abs_VW4: -v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit -v_and_b32 v5, 0x7fffffff, v5 // Remove sign bit -v_and_b32 v6, 0x7fffffff, v6 // Remove sign bit -v_and_b32 v7, 0x7fffffff, v7 // Remove sign bit -s_setpc_b64 s[58:59] -label_Activation_Clippedrelu_VW4: -v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) -v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v5, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v5, s[sgpractivationBeta], v5 // min(x, beta) -v_cndmask_b32 v5, 0.0, v5, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v6, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v6, s[sgpractivationBeta], v6 // min(x, beta) -v_cndmask_b32 v6, 0.0, v6, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v7, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v7, s[sgpractivationBeta], v7 // min(x, beta) -v_cndmask_b32 v7, 0.0, v7, vcc // set x to 0 if <= alpha -s_setpc_b64 s[58:59] -label_Activation_Gelu_VW4: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v5 // k1 * x -v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) -v_mul_f32 v5, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v6 // k1 * x -v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) -v_mul_f32 v6, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v7 // k1 * x -v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) -v_mul_f32 v7, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -s_setpc_b64 s[58:59] -label_Activation_Leakyrelu_VW4: -v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha -v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? -v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v5 // tmp = x * alpha -v_cmp_ge_f32 vcc, v5, 0.0 // x >= 0 ? -v_cndmask_b32 v5, v8, v5, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v6 // tmp = x * alpha -v_cmp_ge_f32 vcc, v6, 0.0 // x >= 0 ? -v_cndmask_b32 v6, v8, v6, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v7 // tmp = x * alpha -v_cmp_ge_f32 vcc, v7, 0.0 // x >= 0 ? -v_cndmask_b32 v7, v8, v7, vcc // set x to tmp if < 0 -s_setpc_b64 s[58:59] -label_Activation_Relu_VW4: -v_max_f32 v4, v4, 0 // x = max(0, x) -v_max_f32 v5, v5, 0 // x = max(0, x) -v_max_f32 v6, v6, 0 // x = max(0, x) -v_max_f32 v7, v7, 0 // x = max(0, x) -s_setpc_b64 s[58:59] -label_Activation_Sigmoid_VW4: -v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // 1 + exp(-x) -v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v5, 0xbfb8aa3b, v5 // (fused -1.442695) -v_exp_f32 v5, v5 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v5, 1.0, v5 // 1 + exp(-x) -v_rcp_f32 v5, v5 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v6, 0xbfb8aa3b, v6 // (fused -1.442695) -v_exp_f32 v6, v6 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v6, 1.0, v6 // 1 + exp(-x) -v_rcp_f32 v6, v6 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v7, 0xbfb8aa3b, v7 // (fused -1.442695) -v_exp_f32 v7, v7 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v7, 1.0, v7 // 1 + exp(-x) -v_rcp_f32 v7, v7 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -s_setpc_b64 s[58:59] -label_Activation_Tanh_VW4: -v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha -v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // e^2x + 1 -v_rcp_f32 v4, v4 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) -v_mul_f32 v5, s[sgpractivationAlpha], v5 // x * alpha -v_mul_f32 v5, 0x4038aa3b, v5 // (fused 2) -v_exp_f32 v5, v5 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v5, 1.0, v5 // e^2x + 1 -v_rcp_f32 v5, v5 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v5, -2.0, v5, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v5, s[sgpractivationBeta], v5 // beta * tanh(x) -v_mul_f32 v6, s[sgpractivationAlpha], v6 // x * alpha -v_mul_f32 v6, 0x4038aa3b, v6 // (fused 2) -v_exp_f32 v6, v6 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v6, 1.0, v6 // e^2x + 1 -v_rcp_f32 v6, v6 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v6, -2.0, v6, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v6, s[sgpractivationBeta], v6 // beta * tanh(x) -v_mul_f32 v7, s[sgpractivationAlpha], v7 // x * alpha -v_mul_f32 v7, 0x4038aa3b, v7 // (fused 2) -v_exp_f32 v7, v7 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v7, 1.0, v7 // e^2x + 1 -v_rcp_f32 v7, v7 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v7, -2.0, v7, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v7, s[sgpractivationBeta], v7 // beta * tanh(x) -s_setpc_b64 s[58:59] -label_Activation_Geluscaling_VW4: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v5 // k1 * x -v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v5, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v6 // k1 * x -v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v6, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v7 // k1 * x -v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v7, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -s_setpc_b64 s[58:59] -label_Activation_Silu_VW4: -v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v5 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v5, v5, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v6 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v6, v6, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v7 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v7, v7, v8 // x / (1 + exp(-x)) -s_setpc_b64 s[58:59] -s_endpgm -label_ASM_End: /// The end of the kernel + +/******************************************/ +/* Begin Kernel */ +/******************************************/ +.amdgcn_target "amdgcn-amd-amdhsa--gfx942" +.text +.protected Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA +.globl Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA +.p2align 8 +.type Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA,@function +.section .rodata,#alloc +.p2align 6 +.amdhsa_kernel Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_accum_offset 256 // accvgpr offset + .amdhsa_next_free_vgpr 512 // vgprs + .amdhsa_next_free_sgpr 84 // sgprs + .amdhsa_group_segment_fixed_size 40960 // lds bytes + .amdhsa_private_segment_fixed_size 0 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_system_sgpr_workgroup_id_y 1 + .amdhsa_system_sgpr_workgroup_id_z 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_user_sgpr_count 13 + .amdhsa_user_sgpr_kernarg_preload_length 11 + .amdhsa_user_sgpr_kernarg_preload_offset 0 +.end_amdhsa_kernel +.text +/* Num VGPR =256 */ +/* Num AccVGPR=256 */ +/* Num SGPR =84 */ + +/******************************************/ +/* Optimizations and Config: */ +/******************************************/ +/* ThreadTile= 16 x 16 */ +/* SubGroup= 16 x 16 */ +/* VectorWidthA=4 */ +/* VectorWidthB=1 */ +/* GlobalReadVectorWidthA=8, GlobalReadVectorWidthB=8 */ +/* DirectToLdsA=False */ +/* DirectToLdsB=False */ +/* UseSgprForGRO=1 */ +.amdgpu_metadata +--- +custom.config: + ProblemType: + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBias: 1 + Activation: True + UseScaleAlphaVec: 1 + UseBeta: True + Batched: True + GroupedGemm: False + SupportUserArgs: True + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: True + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] + 1LDSBuffer: 1 + ScheduleIterAlg: 3 + DepthU: 64 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: 8 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + NoReject: True + InternalSupportParams: + KernArgsVersion: 0 + SupportUserGSU: True + SupportCustomWGM: True + SupportCustomStaggerU: True + UseUniversalArgs: True +amdhsa.version: + - 1 + - 1 +amdhsa.kernels: + - .name: Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA + .symbol: 'Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.kd' + .language: OpenCL C + .language_version: + - 2 + - 0 + .args: + - .name: SizesFree0 + .size: 4 + .offset: 0 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree1 + .size: 4 + .offset: 4 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree2 + .size: 4 + .offset: 8 + .value_kind: by_value + .value_type: u32 + - .name: SizesSum0 + .size: 4 + .offset: 12 + .value_kind: by_value + .value_type: u32 + - .name: Gemm info + .size: 4 + .offset: 16 + .value_kind: by_value + .value_type: u32 + - .name: kernel info + .size: 4 + .offset: 20 + .value_kind: by_value + .value_type: u32 + - .name: D + .size: 8 + .offset: 24 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: C + .size: 8 + .offset: 32 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: A + .size: 8 + .offset: 40 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: B + .size: 8 + .offset: 48 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: strideD0 + .size: 4 + .offset: 56 + .value_kind: by_value + .value_type: u32 + - .name: strideD1 + .size: 4 + .offset: 60 + .value_kind: by_value + .value_type: u32 + - .name: strideC0 + .size: 4 + .offset: 64 + .value_kind: by_value + .value_type: u32 + - .name: strideC1 + .size: 4 + .offset: 68 + .value_kind: by_value + .value_type: u32 + - .name: strideA0 + .size: 4 + .offset: 72 + .value_kind: by_value + .value_type: u32 + - .name: strideA1 + .size: 4 + .offset: 76 + .value_kind: by_value + .value_type: u32 + - .name: strideB0 + .size: 4 + .offset: 80 + .value_kind: by_value + .value_type: u32 + - .name: strideB1 + .size: 4 + .offset: 84 + .value_kind: by_value + .value_type: u32 + - .name: alpha + .size: 4 + .offset: 88 + .value_kind: by_value + .value_type: f32 + - .name: beta + .size: 4 + .offset: 92 + .value_kind: by_value + .value_type: f32 + - .name: AddressScaleAlphaVec + .size: 8 + .offset: 96 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: bias + .size: 8 + .offset: 104 + .value_kind: global_buffer + .value_type: void + .address_space: generic + - .name: biasType + .size: 4 + .offset: 112 + .value_kind: by_value + .value_type: u32 + - .name: StrideBias + .size: 4 + .offset: 116 + .value_kind: by_value + .value_type: u32 + - .name: activationAlpha + .size: 4 + .offset: 120 + .value_kind: by_value + .value_type: f32 + - .name: activationBeta + .size: 4 + .offset: 124 + .value_kind: by_value + .value_type: f32 + - .name: activationType + .size: 4 + .offset: 128 + .value_kind: by_value + .value_type: u32 + .group_segment_fixed_size: 40960 + .kernarg_segment_align: 8 + .kernarg_segment_size: 136 + .max_flat_workgroup_size: 256 + .private_segment_fixed_size: 0 + .sgpr_count: 84 + .sgpr_spill_count: 0 + .vgpr_count: 256 + .vgpr_spill_count: 0 + .wavefront_size: 64 +... +.end_amdgpu_metadata +Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA: +label_ASM_Start: /// Main body of the asm kernel + +/* Magic div and mod functions */ +.macro V_MAGIC_DIV dstIdx:req dividend:req magicNumber:req magicShift:req magicA:req + v_mul_hi_u32 v[\dstIdx+1] \dividend \magicNumber + v_mul_lo_u32 v[\dstIdx+0] \dividend \magicA + v_add_u32 v[\dstIdx+0] v[\dstIdx+0] v[\dstIdx+1] + v_lshrrev_b32 v[\dstIdx+0] \magicShift v[\dstIdx+0] +.endm + +/******************************************/ +/* VGPR Assignments */ +/******************************************/ +/* ValuC range: [0-0), serializedStore enabled */ +.set vgprValuC, 0 +/* ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx */ +.set vgprValuA_X0_I0_0, 0 +.set vgprValuA_X2_I0_0, 16 +.set vgprValuB_X0_I0, 32 +.set vgprValuB_X2_I0, 96 +.set vgprLocalWriteAddrA, 160 +.set vgprLocalWriteAddrB, 161 +.set vgprGlobalReadOffsetA, 162 +.set vgprGlobalReadOffsetB, 163 +.set vgprG2LB, 164 +.set vgprValuA_X0_I0_1, 196 +.set vgprValuA_X2_I0_1, 212 +.set vgprLocalReadAddrA, 228 +.set vgprLocalReadAddrB, 229 +.set vgprSerial, 230 + +/******************************************/ +/* SGPR Assignments */ +/******************************************/ +.set sgprKernArgAddress, 0 +.set sgprWorkGroup0, 2 +.set sgprWorkGroup1, 3 +.set sgprWorkGroup2, 4 +.set sgprArgType, 5 +.set sgprGSUSumIdx, 6 +.set sgprGSULog2BpeC, 8 +.set sgprGSULog2BpeD, 9 +.set sgprStaggerU, 10 +.set sgprWGM, 11 +.set sgprLoopCounterL, 12 +.set sgprOrigLoopCounter, 13 +.set sgprSrdD, 16 +.set sgprSrdC, 20 +.set sgprNumWorkGroups0, 14 +.set sgprNumWorkGroups1, 15 +.set sgprSizesFree, 24 +.set sgprSizesSum, 27 +.set sgprAddressD, 28 +.set sgprAddressC, 30 +.set sgprAddressA, 32 +.set sgprAddressB, 34 +.set sgprStridesD, 36 +.set sgprStridesC, 38 +.set sgprStridesA, 40 +.set sgprStridesB, 42 +.set sgprAlpha, 44 +.set sgprBeta, 45 +.set sgprGSU, 46 + +/* Size Assignments */ +.set sgprSizeI, sgprSizesFree+0 +.set sgprSizeJ, sgprSizesFree+1 +.set sgprSizeK, sgprSizesFree+2 +.set sgprSizeL, sgprSizesSum+0 + +/* Stride Assignments */ +.set constStrideD0I, 1 +.set sgprStrideD1J, sgprStridesD+0 +.set sgprStrideDK, sgprStridesD+1 +.set constStrideC0I, 1 +.set sgprStrideC1J, sgprStridesC+0 +.set sgprStrideCK, sgprStridesC+1 +.set constStrideAL, 1 +.set sgprStrideA0I, sgprStridesA+0 +.set sgprStrideAK, sgprStridesA+1 +.set constStrideBL, 1 +.set sgprStrideB1J, sgprStridesB+0 +.set sgprStrideBK, sgprStridesB+1 + +.set MT0, 256 +.set MT1, 256 +.set DepthU, 64 +.set BpeA, 2 +.set BpeALog2, 1 +.set BpeB, 2 +.set BpeBLog2, 1 +.set BpeAGR, 2 +.set BpeAGRLog2, 1 +.set BpeBGR, 2 +.set BpeBGRLog2, 1 +/* Number of elements to shift-left SRD */ +.set SrdShiftLeftA, 8 +.set SrdShiftLeftB, 8 +/* 2GB limit - set offsets to -1 to exceed this and clamp */ +.set BufferLimit, 0xffffffff +.set BufferOOB, 0x80000000 + +/******************************************/ +/* Bits 127:96 of SRD. */ +/* hex: 0x00020000 */ +/* dst_sel_x (3b): 0 */ +/* dst_sel_y (3b): 0 */ +/* dst_sel_z (3b): 0 */ +/* dst_sel_w (3b): 0 */ +/* num_format (3b): 0 */ +/* data_format (4b): 4 */ +/* user_vm_enable (1b): 0 */ +/* user_vm_mode (1b): 0 */ +/* index_stride (2b): 0 */ +/* add_tid_enable (1b): 0 */ +/* _unusedA (3b): 0 */ +/* nv (1b): 0 */ +/* _unusedB (2b): 0 */ +/* type (2b): 0 */ +/******************************************/ +.set Srd127_96, 0x00020000 + +/* Global Offset A */ +.macro GLOBAL_OFFSET_A vgprAddr:req vgprTmp:req + v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) + v_and_b32 v[\vgprAddr+0], 15, v[\vgprTmp+0] // 1. M offset: mIdx = wtid % MI_M(16) + v_mul_lo_u32 v[\vgprAddr+0], s[sgprStrideA0I], v[\vgprAddr+0] // 1. M offset: mOffset = mIdx * mStride(k) + v_lshlrev_b32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) + v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) + v_lshrrev_b32 v[\vgprTmp+0], 4, v[\vgprTmp+0] // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) + v_lshlrev_b32 v[\vgprTmp+0], 0x3, v[\vgprTmp+0] // 5. K offset: lrKOffset = kIdx * mStride(8) + v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 6. offset in wave: lrOffset = bnOffset + lrKOffset + v_lshrrev_b32 v[\vgprTmp+0], 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) + v_and_b32 v[\vgprTmp+0], 3, v[\vgprTmp+0] // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) + v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideA0I], v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) + v_lshlrev_b32 v[\vgprTmp+0], 0x6, v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) + v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 7. final local read offset: flrOffset = lrOffset + WOffset + v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift + v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element +.endm + +/* Global Offset B */ +.macro GLOBAL_OFFSET_B vgprAddr:req vgprOffsetL:req vgprOffset1J:req vgprTmp:req + v_mul_lo_u32 v[\vgprTmp+0] s[sgprStrideB1J] v[\vgprOffset1J] // mul d1 lower + v_add_co_u32 v[\vgprAddr+0] vcc v[\vgprOffsetL] v[\vgprTmp+0] // accumulate K lower + v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift + v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element +.endm + +/* Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor; */ +.macro DYNAMIC_VECTOR_DIVIDE vQuotient vRemainder vDividend vDivisor vTmp0 vTmp1 sTmp + v_cvt_f32_u32 v[\vQuotient] v[\vDivisor] + v_rcp_f32 v[\vQuotient] v[\vQuotient] + v_mul_f32 v[\vQuotient] 0x4f800000 v[\vQuotient] + v_cvt_u32_f32 v[\vQuotient] v[\vQuotient] + v_mul_lo_u32 v[\vRemainder] v[\vDivisor] v[\vQuotient] + v_mul_hi_u32 v[\vTmp0] v[\vDivisor] v[\vQuotient] + v_sub_co_u32 v[\vTmp1] vcc 0x0 v[\vRemainder] + v_cmp_ne_i32 s[\sTmp:\sTmp+1] 0x0 v[\vTmp0] + v_cndmask_b32 v[\vRemainder] v[\vTmp1] v[\vRemainder] s[\sTmp:\sTmp+1] + v_mul_hi_u32 v[\vRemainder] v[\vRemainder] v[\vQuotient] + v_sub_co_u32 v[\vTmp0] vcc v[\vQuotient] v[\vRemainder] + v_add_co_u32 v[\vQuotient] vcc v[\vQuotient] v[\vRemainder] + v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vTmp0] s[\sTmp:\sTmp+1] + v_mul_hi_u32 v[\vQuotient] v[\vQuotient] v[\vDividend] + v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] + v_sub_co_u32 v[\vTmp0] vcc v[\vDividend] v[\vRemainder] + v_cmp_ge_u32 s[\sTmp:\sTmp+1] v[\vDividend] v[\vRemainder] + v_add_co_u32 v[\vRemainder] vcc 0x1 v[\vQuotient] + v_add_co_u32 v[\vTmp1] vcc -1 v[\vQuotient] + v_cmp_le_u32 vcc v[\vDivisor] v[\vTmp0] + s_and_b64 vcc s[\sTmp:\sTmp+1] vcc + v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vRemainder] vcc + v_cndmask_b32 v[\vQuotient] v[\vTmp1] v[\vQuotient] s[\sTmp:\sTmp+1] + v_cmp_ne_i32 vcc 0x0 v[\vDivisor] + v_cndmask_b32 v[\vQuotient] -1 v[\vQuotient] vcc // final result + v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] + v_sub_co_u32 v[\vRemainder] vcc v[\vDividend] v[\vRemainder] // final result +.endm + +/******************************************/ +/* Allocate Resources */ +/******************************************/ + +/* Load num of Gemms */ +s_load_dword s47, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 + +/* Load GSU data */ +s_load_dword s[sgprGSU], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x4 +s_waitcnt lgkmcnt(0) +s_lshr_b32 s48, s47, 0x1e // Get arg type +s_and_b32 s47, 0x3fffffff, s47 // Get nums of gemm +s_cmp_eq_u32 s48, 0 // Is kernel args +s_cbranch_scc0 label_HBMArgs +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 +s_load_dwordx4 s[40:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x40 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_waitcnt lgkmcnt(0) +s_branch label_LoadArgsEnd +label_HBMArgs: + +/* Load address of kernel arguments */ +s_load_dwordx2 s[sgprKernArgAddress:sgprKernArgAddress+1], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8 +s_waitcnt lgkmcnt(0) // wait for args to load +label_LoadArgsEnd: +s_branch label_common_kernel_entry + +/* pad 41 snops to satisfy 0x100 code size for Preload Backward Compatibility Prologue */ +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +label_Preload_Offset_Start: +s_and_b32 s47, 0x3fffffff, s2 // Get nums of gemm +s_lshr_b32 s48, s2, 0x1e // Get arg type +s_mov_b32 s[sgprGSU], s3 // Preload internal args +s_cmp_eq_u32 s48, 0 // Is kernel args +s_cbranch_scc0 label_Preload_HBMArgs +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dword s33, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x24 +s_load_dwordx2 s[34:35], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x28 +s_load_dwordx8 s[36:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x30 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_mov_b32 s24, s4 // move preload data to correct sgpr +s_mov_b32 s25, s5 // move preload data to correct sgpr +s_mov_b32 s26, s6 // move preload data to correct sgpr +s_mov_b32 s27, s7 // move preload data to correct sgpr +s_mov_b32 s28, s8 // move preload data to correct sgpr +s_mov_b32 s29, s9 // move preload data to correct sgpr +s_mov_b32 s30, s10 // move preload data to correct sgpr +s_mov_b32 s31, s11 // move preload data to correct sgpr +s_mov_b32 s32, s12 // move preload data to correct sgpr +s_branch label_Preload_LoadArgsEnd +label_Preload_HBMArgs: +s_mov_b64 s[sgprKernArgAddress:sgprKernArgAddress+1], s[4:5] // Load address of kernel arguments +label_Preload_LoadArgsEnd: +label_common_kernel_entry: /// for both preload/non-preload common code +s_mov_b32 s[sgprWorkGroup0+0], s13 // restore workgroup id +s_mov_b32 s[sgprWorkGroup0+1], s14 // restore workgroup id +s_mov_b32 s[sgprWorkGroup0+2], s15 // restore workgroup id +s_and_b32 s[sgprWGM], s[sgprGSU], 0xff00 // Restore WGM +s_lshr_b32 s[sgprWGM], s[sgprWGM], 0x8 +s_and_b32 s[sgprStaggerU], s[sgprGSU], 0xffff0000 // Restore StaggerU related vars +s_lshr_b32 s[sgprStaggerU], s[sgprStaggerU], 0x10 +s_and_b32 s[sgprGSU], s[sgprGSU], 0xff // Restore GSU +s_mov_b32 s[sgprArgType], s48 +s_mov_b32 m0, 0xa000 // LDS clamp at 40960 bytes +v_mov_b32 v[vgprSerial], v0 // thread serial id +s_cmp_eq_u32 s48, 0 +s_cbranch_scc0 label_MultiGemm +/* init: add vgpr [0...160) to pool */ +/* init: add vgpr [0...0) to pool */ +/* init: add agpr [0...256) to pool */ + +/******************************************/ +/* Local Read Addresses */ +/******************************************/ + +/* local read addresses: tile assignments a/b */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset + +/* local read addresses: final offsets a */ +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 + +/* local read addresses: final offsets b */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 + +/* local read addresses: declare addresses a */ +/* N/A */ + +/* local read addresses: declare addresses b */ + +/******************************************/ +/* Local Write Addresses */ +/******************************************/ +/* LVCA = 8 */ +/* v1 = A-unroll = serial%LVCA */ +v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 +v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 +v_mov_b32 v4, v1 // copy for GlobalSplitU +/* LVCB = 8 */ +/* v3 = B-unroll = serial%LVCB */ +v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 +v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 +v_mov_b32 v5, v3 // copy for GlobalSplitU +/* lwaUnrollAssignmentA = v4 */ +/* lwaUnrollAssignmentB = v5 */ + +/* local write addresses: first offset a */ +v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 +v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 + +/* local write addresses: first offset b */ +v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 +v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 +v_mov_b32 v8, MT0 // set MT0 into sgpr +v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +v_mov_b32 v8, MT1 // set MT1 into sgpr +v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size +v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +s_nop 0 // 1 wait states +v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 +s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args +s_branch label_MultiGemmEnd +label_MultiGemm: + +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_IsExternalValid // branch if ArgType == 2 +s_mov_b32 s15, 124 +s_mul_i32 s54, s47, 4 +s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] +s_branch label_IsExternalValidEnd +label_IsExternalValid: +s_mov_b32 s15, 196 +s_mov_b32 s54, 0x0 +s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] +label_IsExternalValidEnd: + +/* Grouped Gemm:: prefetch 1 arg load */ +s_mov_b32 s14, 1 +s_mov_b32 s55, 0 +s_load_dwordx4 s[24:27], s[48:49], s54 +s_cmpk_eq_u32 s47, 1 // if gemm_count is 1? +s_cbranch_scc1 label_wgTable_noLoadLoop + +/* Grouped Gemm:: accumulate numTiles for each gemm */ +/* Grouped Gemm:: loop start */ +label_Loop_GemmCount: +s_waitcnt lgkmcnt(0) +s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 +s_and_b32 s50, 255, s24 // s50 = s24 % 256 +s_addc_u32 s52, s52, 0x0 +s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 +s_and_b32 s50, 255, s25 // s50 = s25 % 256 +s_addc_u32 s53, s53, 0x0 +s_mul_i32 s52, s52, s53 +s_mul_i32 s52, s52, s26 +s_mul_i32 s52, s52, s[sgprGSU] +s_add_u32 s55, s55, s52 +s_cmp_lt_u32 s[sgprWorkGroup0], s55 +s_cbranch_scc1 label_FOUND +s_add_u32 s54, s54, s15 +s_load_dwordx4 s[24:27], s[48:49], s54 +s_add_u32 s14, s14, 1 +s_cmp_lt_u32 s14, s47 +s_cbranch_scc1 label_Loop_GemmCount + +/* Grouped Gemm:: noLoadLoop */ +label_wgTable_noLoadLoop: +s_waitcnt lgkmcnt(0) +s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 +s_and_b32 s50, 255, s24 // s50 = s24 % 256 +s_addc_u32 s52, s52, 0x0 +s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 +s_and_b32 s50, 255, s25 // s50 = s25 % 256 +s_addc_u32 s53, s53, 0x0 +s_mul_i32 s52, s52, s53 +s_mul_i32 s52, s52, s26 +s_mul_i32 s52, s52, s[sgprGSU] +s_add_u32 s55, s55, s52 + +/* Grouped Gemm:: gemmIndex found */ +label_FOUND: +s_sub_u32 s49, s14, 1 +s_sub_u32 s48, s55, s52 +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalStruct // branch if ArgType == 2 + +/* Grouped Gemm: offset argument address to gemm */ +/* Grouped Gemm: offset address from wg_table_start to args_start */ +s_lshl2_add_u32 s[sgprKernArgAddress], s47, s[sgprKernArgAddress] +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 +/* Grouped Gemm: offset address from args_start to gemm_start */ +s_mul_i32 s49, s49, 124 +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_branch label_LoadExternalStructEnd +label_LoadExternalStruct: +/* Grouped Gemm: offset address from args_start to gemm_start */ +s_mul_i32 s49, s49, 196 +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 +s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 +s_load_dword s44, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +// Read Beta +s_load_dword s45, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x60 +label_LoadExternalStructEnd: +/* init: add vgpr [0...160) to pool */ +/* init: add vgpr [0...0) to pool */ +/* init: add agpr [0...256) to pool */ + +/******************************************/ +/* Local Read Addresses */ +/******************************************/ + +/* local read addresses: tile assignments a/b */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset + +/* local read addresses: final offsets a */ +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 + +/* local read addresses: final offsets b */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 + +/* local read addresses: declare addresses a */ +/* N/A */ + +/* local read addresses: declare addresses b */ + +/******************************************/ +/* Local Write Addresses */ +/******************************************/ +/* LVCA = 8 */ +/* v1 = A-unroll = serial%LVCA */ +v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 +v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 +v_mov_b32 v4, v1 // copy for GlobalSplitU +/* LVCB = 8 */ +/* v3 = B-unroll = serial%LVCB */ +v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 +v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 +v_mov_b32 v5, v3 // copy for GlobalSplitU +/* lwaUnrollAssignmentA = v4 */ +/* lwaUnrollAssignmentB = v5 */ + +/* local write addresses: first offset a */ +v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 +v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 + +/* local write addresses: first offset b */ +v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 +v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 +v_mov_b32 v8, MT0 // set MT0 into sgpr +v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +v_mov_b32 v8, MT1 // set MT1 into sgpr +v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size +v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +s_nop 0 // 1 wait states +v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 +s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args + +/* Early stop if N(SizeFreeJ) == 0 */ +s_cmp_eq_u32 s[sgprSizeJ], 0x0 +s_cbranch_scc0 label_NoEarlyStop_N0 +label_EarlyStop_if_N_is_0: +s_endpgm +label_NoEarlyStop_N0: + +/* Grouped Gemm: remap wg from 1D(idxWG012) to 3D(wg2,wg1,wg0) */ +/* wg2 = idxWG012 * smallMagicNumber(1/(numWG0*numWG1)) */ +s_mul_i32 s48, s[sgprNumWorkGroups0], s[sgprNumWorkGroups1] +s_mul_i32 s48, s48, s[sgprGSU] +v_cvt_f32_u32 v6, s48 // s48 = s[sgprWorkGroup0] / s48 +v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 +v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s48 +v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s48 +v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 +v_mul_u32_u24 v7, v6, s48 // s48 = s[sgprWorkGroup0] / s48 +v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s48 +v_cmpx_eq_u32 exec, v7, s48 // s48 = s[sgprWorkGroup0] / s48 +v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s48 +s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s48 +v_readfirstlane_b32 s48, v6 +s_mov_b32 s[sgprWorkGroup2], s48 +/* idxWG01 = idxWG012 - wg2 * numWG0 * numWG1 */ +s_mul_i32 s48, s[sgprNumWorkGroups1], s[sgprNumWorkGroups0] +s_mul_i32 s48, s48, s[sgprWorkGroup2] +s_mul_i32 s48, s48, s[sgprGSU] +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 +/* wg1 = idxWG01 * smallMagicNumber(1/numWG0) */ +v_cvt_f32_u32 v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_readfirstlane_b32 s48, v6 +s_mov_b32 s[sgprWorkGroup1], s48 +/* wg0 = idxWG01 - wg1 * numWG0 */ +s_mul_i32 s48, s[sgprWorkGroup1], s[sgprNumWorkGroups0] +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 + +/* Early stop if wg exceed */ +s_cmp_ge_u32 s[sgprWorkGroup2], s[sgprSizesFree+2] +s_cbranch_scc0 label_NoEarlyStop_wgExceed +label_EarlyStop_if_wg_exceed: +s_endpgm +label_NoEarlyStop_wgExceed: + +label_MultiGemmEnd: +.set sgprSrdA, 48 +.set sgprSrdB, 52 +.set sgprShadowLimitA, 56 +.set sgprShadowLimitB, 58 +.set sgprStaggerUIter, 47 +.set sgprWrapUA, 60 +.set sgprWrapUB, 62 +.set sgprGlobalReadIncsA, 64 +.set sgprGlobalReadIncsB, 65 +.set sgprScalarGlobalReadOffsetA, 66 +.set sgprScalarGlobalReadOffsetB, 73 +s_sub_u32 s[sgprAddressA+0], s[sgprAddressA+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprAddressA+1], s[sgprAddressA+1], 0 // pre-pad to make room for possible pointer shift +s_sub_u32 s[sgprAddressB+0], s[sgprAddressB+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprAddressB+1], s[sgprAddressB+1], 0 // pre-pad to make room for possible pointer shift + +/* Short circuit condition if Alpha == 0, then sumDims=0 */ +v_cmp_eq_f32 vcc, s[sgprAlpha], 0.0 // s[Alpha] == 0.0f ? +s_cbranch_vccz label_AlphaNonZero // branch if s[Alpha] != 0 +s_mov_b32 s[sgprSizesSum+0], 0x0 // Set summation dim=0 if Alpha == 0 +label_AlphaNonZero: + +/******************************************/ +/* Begin setupNewTile */ +/******************************************/ + +/* global read addresses: work-group */ +/* graWorkGroup mapping */ +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU // branch if GSU == 1 +// GSU-not-WGMapRR :nwg1 = (size1J + MT1J - 1) / MT1J; +v_cvt_f32_u32 v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cvt_f32_u32 v7, s[sgprWorkGroup1] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mul_f32 v6, v6, v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mul_u32_u24 v7, v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_sub_u32 v7, s[sgprWorkGroup1], v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cmpx_eq_u32 exec, v7, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_add_u32 v6, 1, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mov_b32 v7, 0 // s[sgprGSUSumIdx] = s[sgprWorkGroup1] % s[sgprGSU] +s_mov_b64 exec, -1 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_readfirstlane_b32 s[sgprWorkGroup1], v6 +v_readfirstlane_b32 s[sgprGSUSumIdx], v7 +s_mov_b32 s[sgprGSULog2BpeC], 1 +s_mov_b32 s[sgprGSULog2BpeD], 2 +s_branch label_GSU_End +label_GSU: +s_mov_b64 s[sgprGSUSumIdx:sgprGSUSumIdx+1], 0 // Set GSUSumIdx to 0 +s_mov_b32 s[sgprGSULog2BpeC], 1 +s_mov_b32 s[sgprGSULog2BpeD], 1 +label_GSU_End: +s_cmp_le_u32 s[sgprWGM], 1 // WGM <= 1 ? +s_cbranch_scc1 label_WGM // branch if WGM <= 1 +v_cvt_f32_u32 v6, s[sgprWGM] // WGM +v_rcp_iflag_f32 v6, v6 // WGM +v_cvt_f32_u32 v7, s[sgprWorkGroup1] // WGM +v_mul_f32 v6, v6, v7 // WGM +v_cvt_u32_f32 v6, v6 // WGM +v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM +v_sub_u32 v7, s[sgprWorkGroup1], v7 // WGM +v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM +v_add_u32 v6, 1, v6 // WGM +s_mov_b64 exec, -1 // WGM +v_readfirstlane_b32 s82, v6 +s_mul_i32 s83, s82, s[sgprWGM] // quotient * non-magic divisor +s_sub_u32 s83, s[sgprWorkGroup1], s83 // WorkGroup1=remainder +s_mul_i32 s83, s83, s[sgprNumWorkGroups0] // (wg1 % WGM)*nwg0 +s_add_u32 s83, s83, s[sgprWorkGroup0] // wgSerial = wg0 + (wg1 % WGM)*nwg0 +v_cvt_f32_u32 v6, s[sgprWGM] // WGM +v_rcp_iflag_f32 v6, v6 // WGM +v_cvt_f32_u32 v7, s[sgprNumWorkGroups1] // WGM +v_mul_f32 v6, v6, v7 // WGM +v_cvt_u32_f32 v6, v6 // WGM +v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM +v_sub_u32 v7, s[sgprNumWorkGroups1], v7 // WGM +v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM +v_add_u32 v6, 1, v6 // WGM +s_mov_b64 exec, -1 // WGM +v_readfirstlane_b32 s80, v6 +s_mul_i32 s81, s[sgprWGM], s80 // quotient * non-magic divisor +s_sub_u32 s81, s[sgprNumWorkGroups1], s81 // WorkGroup1=remainder +s_cmp_eq_u32 s81, 0 // remainder == 0 ? +s_cmov_b32 s81, s[sgprWGM] // remainder = WGM if remainder == 0 +s_cmp_ge_u32 s82, s80 // blockId >= numFullBlocks ? +s_cselect_b32 s80, s81, s[sgprWGM] +v_cvt_f32_u32 v6, s80 // s[sgprWorkGroup0] = s83 / s80 +v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 +v_cvt_f32_u32 v7, s83 // s[sgprWorkGroup0] = s83 / s80 +v_mul_f32 v6, v6, v7 // s[sgprWorkGroup0] = s83 / s80 +v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 +v_mul_u32_u24 v7, v6, s80 // s[sgprWorkGroup0] = s83 / s80 +v_sub_u32 v7, s83, v7 // s[sgprWorkGroup0] = s83 / s80 +v_cmpx_eq_u32 exec, v7, s80 // s[sgprWorkGroup0] = s83 / s80 +v_add_u32 v6, 1, v6 // s[sgprWorkGroup0] = s83 / s80 +v_mov_b32 v7, 0 // s[sgprWorkGroup1] = s83 % s80 +s_mov_b64 exec, -1 // s[sgprWorkGroup0] = s83 / s80 +v_readfirstlane_b32 s[sgprWorkGroup0], v6 +v_readfirstlane_b32 s[sgprWorkGroup1], v7 +s_mul_i32 s82, s82, s[sgprWGM] // blockId * WGM +s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s82 // wg1 += blockId * WGM +label_WGM: + +/* global read addresses: tile offset assignment a */ +/* graTileAssignmentA = v0 */ + +/* global read addresses: tile offset assignment b */ +/* graTileAssignmentB = v2 */ + +/* global read addresses: unroll assignment a */ +/* v1 */ + +/* global read addresses: unroll assignment b */ +/* v3 */ + +/* global read addresses: other free assignments */ +/* s[sgprWorkGroup2] */ + +/* global read addresses: tile offsets a */ + +/* global read addresses: tile offsets b */ + +/* global read addresses: unroll offsets a */ + +/* global read addresses: unroll offsets b */ + +/* global read addresses: final offsets a */ +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+0, 6 +s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 32 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] + +/* global read addresses: final offsets b */ +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+0, 3, 2, 6 // gROB_0_0_0_0 +s_mul_i32 s[sgprScalarGlobalReadOffsetB+0], s[sgprStrideB1J], 32 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+0], s[sgprScalarGlobalReadOffsetB+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+1], s[sgprStrideB1J], 64 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+1], s[sgprScalarGlobalReadOffsetB+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+2], s[sgprStrideB1J], 96 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+2], s[sgprScalarGlobalReadOffsetB+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+3], s[sgprStrideB1J], 128 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+3], s[sgprScalarGlobalReadOffsetB+3], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+4], s[sgprStrideB1J], 160 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+4], s[sgprScalarGlobalReadOffsetB+4], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+5], s[sgprStrideB1J], 192 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+5], s[sgprScalarGlobalReadOffsetB+5], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+6], s[sgprStrideB1J], 224 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+6], s[sgprScalarGlobalReadOffsetB+6], 0x1 // scalar offset *= bytes/element + +/* global read addresses: addresses a */ +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s83, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT +s_mul_i32 s82, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT +s_mul_hi_u32 s83, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride +s_mul_i32 s82, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride +s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart +s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart +s_mov_b32 s[sgprShadowLimitA+0], 1 // Init tensor size +s_mov_b32 s[sgprShadowLimitA+1], 0 // init tensor size +s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) +s_mul_hi_u32 s81, constStrideAL, s80 // stride x (size-1) +s_mul_i32 s80, constStrideAL, s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size +s_sub_u32 s80, s[sgprSizeI], 1 // (size-1) +s_mul_hi_u32 s81, s[sgprStrideA0I], s80 // stride x (size-1) +s_mul_i32 s80, s[sgprStrideA0I], s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s82 // sub tileStart +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s83 // sub tileStart +s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], 0x1 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s81, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s80, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s82, s82, s80 // accum wg term to tilestart +s_addc_u32 s83, s83, s81 // accum wg term to tilestart +s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE +s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s82 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s83 // SRD base = Address+ tileStart1 +s_mov_b32 s[sgprSrdA+3], Srd127_96 // Set bits 127_96 in SRD + +/* global read addresses: addresses b */ +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s83, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT +s_mul_i32 s82, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT +s_mul_hi_u32 s83, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_mul_i32 s82, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart +s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart +s_mov_b32 s[sgprShadowLimitB+0], 1 // Init tensor size +s_mov_b32 s[sgprShadowLimitB+1], 0 // init tensor size +s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) +s_mul_hi_u32 s81, constStrideBL, s80 // stride x (size-1) +s_mul_i32 s80, constStrideBL, s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size +s_sub_u32 s80, s[sgprSizeJ], 1 // (size-1) +s_mul_hi_u32 s81, s[sgprStrideB1J], s80 // stride x (size-1) +s_mul_i32 s80, s[sgprStrideB1J], s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s82 // sub tileStart +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s83 // sub tileStart +s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], 0x1 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s81, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s80, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s82, s82, s80 // accum wg term to tilestart +s_addc_u32 s83, s83, s81 // accum wg term to tilestart +s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE +s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s82 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s83 // SRD base = Address+ tileStart1 +s_mov_b32 s[sgprSrdB+3], Srd127_96 // Set bits 127_96 in SRD +s_mul_i32 s80, s[sgprGSU], DepthU*BpeAGR +s_mov_b32 s[sgprGlobalReadIncsA+0], s80 // incrA (unrollIdx) + +/* global read addresses: increments b */ +s_mul_i32 s80, s[sgprGSU], DepthU*BpeBGR +s_mov_b32 s[sgprGlobalReadIncsB+0], s80 // incrB (unrollIdx) + +/* declare loop num iterations */ +s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], 6 // s[sgprLoopCounterL] = s[sgprSizesSum+0] / 64 +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_1 // branch if GSU == 1 +v_cvt_f32_u32 v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_rcp_iflag_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cvt_f32_u32 v1, s[sgprLoopCounterL] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mul_f32 v0, v0, v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cvt_u32_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mul_u32_u24 v1, v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_sub_u32 v1, s[sgprLoopCounterL], v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cmpx_eq_u32 exec, v1, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_add_u32 v0, 1, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mov_b32 v1, 0 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSU] +s_mov_b64 exec, -1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_readfirstlane_b32 s[sgprLoopCounterL], v0 +v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1 +s_add_u32 s80, 1, s[sgprLoopCounterL] // tmp<-numIterMyWg+ +s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx < numIterPerWgRemainder +s_cmov_b32 s[sgprLoopCounterL], s80 // numIterMyWg++ if needed +label_GSU_1: +s_mov_b32 s[sgprOrigLoopCounter], s[sgprLoopCounterL] // copy loop counter +s_and_b32 s82, s[sgprStaggerU], 0x1f00 +s_lshr_b32 s82, s82, 0x8 +s_and_b32 s83, s[sgprStaggerU], 0xe000 +s_and_b32 s[sgprStaggerU], s[sgprStaggerU], 0xff +s_mov_b32 s80, s[sgprStaggerU] // init staggerU +label_beginStaggerUIter: +s_lshl_b32 s81, s80, s82 // shift by StaggerUStride +s_cmp_ge_u32 s[sgprOrigLoopCounter], s81 // loopCount >= current shift Count +s_cbranch_scc1 label_endStaggerUIter // jump to end +s_lshr_b32 s80, s80, 1 // step down to smaller stagger +s_branch label_beginStaggerUIter // jump to begin +label_endStaggerUIter: +s_sub_u32 s81, s80, 1 // staggerU mask +s_cmp_ge_u32 s80, 1 // if current staggerU >= 1 +s_cselect_b32 s[sgprStaggerUIter], s81, 0 // set Mask +s_cmp_eq_u32 s83, 0x0 +s_cbranch_scc1 label_StaggerUMapping_1 +s_mov_b32 s80, s[sgprWorkGroup0] +s_branch label_staggerInputEnd +label_StaggerUMapping_1: +s_cmp_eq_u32 s83, 0x2000 +s_cbranch_scc1 label_StaggerUMapping_2 +s_mov_b32 s80, s[sgprWorkGroup1] +s_branch label_staggerInputEnd +label_StaggerUMapping_2: +s_cmp_eq_u32 s83, 0x4000 +s_cbranch_scc1 label_StaggerUMapping_3 +s_mov_b32 s80, -0x1 +s_branch label_staggerInputEnd +label_StaggerUMapping_3: +s_cmp_eq_u32 s83, 0x6000 +s_cbranch_scc1 label_StaggerUMapping_4 +s_mul_i32 s81, s[sgprNumWorkGroups0], s[sgprWorkGroup1] +s_add_u32 s80, s80, s81 +s_add_u32 s80, s80, s[sgprWorkGroup0] +s_branch label_staggerInputEnd +label_StaggerUMapping_4: +s_cmp_eq_u32 s83, 0x8000 +s_cbranch_scc1 label_staggerInputEnd +s_mov_b32 s80, -0x1 +s_branch label_staggerInputEnd +label_staggerInputEnd: +s_and_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s80 // Compute actual stagger start for this tile +s_lshl_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s82 // shift by StaggerUStride + +/* SRDs += (StaggerUIter) * GlobalReadIncsA+0 */ +s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUA+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUA+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0], s[sgprWrapUA+0] // remove one iteration +s_subb_u32 s[sgprWrapUA+1], 0, s[sgprWrapUA+1] // remove one iteration +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* SRDs += (StaggerUIter) * GlobalReadIncsB+0 */ +s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUB+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUB+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0], s[sgprWrapUB+0] // remove one iteration +s_subb_u32 s[sgprWrapUB+1], 0, s[sgprWrapUB+1] // remove one iteration +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +s_add_u32 s[sgprStaggerUIter], s[sgprStaggerUIter], 2 // Subtract (PGR-1); StaggerUIter now contains target iteration to wrap +/* local read addresses: init pointers a */ + +/* localReadInitPointers */ +/* local read addresses: init pointers b */ + +/* localReadInitPointers */ + +/* prefetch: global -> local */ +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? +s_cbranch_scc1 label_ShadowInitStart // skip to ShadowInitStart iter b/c numIter==0 +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+0:vgprValuA_X2_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+4:vgprValuA_X2_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+8:vgprValuA_X2_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+12:vgprValuA_X2_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 + +/* global read inc A loopL */ +s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* global read inc B loopL */ +s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 + +/******************************************/ +/* End setupNewTile */ +/******************************************/ +label_ShadowInitStart: +s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdD+2], 0x80000000 +s_mov_b32 s[sgprSrdD+3], Srd127_96 // Set bits 127_96 in post-loop SRD + +s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdC+2], 0x80000000 +s_mov_b32 s[sgprSrdC+3], Srd127_96 // Set bits 127_96 in post-loop SRD + + +s_mul_i32 s82, MT1, s[sgprWorkGroup1] // <- wg1*MT1 +s_mul_hi_u32 s81, s82, s[sgprStrideC1J] // ScaleC s82 by Stride +s_mul_i32 s80, s82, s[sgprStrideC1J] // ScaleC s82 by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprAddressC+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprAddressC+1], s81 // add hi to SRD +s_mul_hi_u32 s81, s82, s[sgprStrideD1J] // ScaleD s82 by Stride +s_mul_i32 s80, s82, s[sgprStrideD1J] // ScaleD s82 by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprAddressD+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprAddressD+1], s81 // add hi to SRD + +s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride +s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s81 // add hi to SRD +s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride +s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi to SRD + +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_2 // branch if GSU == 1 +// GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s +s_mul_hi_u32 s81, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 +s_mul_i32 s80, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 +s_sub_u32 s82, s[sgprSizesFree+1], 1 // Free1 +s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free1 +s_mul_hi_u32 s83, s82, s[sgprStrideC1J] // Free1 +s_mul_i32 s82, s82, s[sgprStrideC1J] // Free1 +s_add_u32 s80, s80, s82 // Free1 +s_addc_u32 s81, s81, s83 // Free1 +s_sub_u32 s82, s[sgprSizesFree+2], 1 // Free2 +s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free2 +s_mul_hi_u32 s83, s82, s[sgprStrideCK] // Free2 +s_mul_i32 s82, s82, s[sgprStrideCK] // Free2 +s_add_u32 s80, s80, s82 // Free2 +s_addc_u32 s81, s81, s83 // Free2 +s_lshl_b64 s[80:81], s[80:81], 2 // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo GSU offset to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi GSU offset to SRD +label_GSU_2: +.set sgprGSULog2BpeC, UNDEF + +/* initC: remove ValuC vgpr buffer [0...0) from pool */ + +/* initC: remove acc vgpr buffer [0...256) from pool */ + +/* initC: remove ValuA/B vgpr buffer [0...160) from pool */ +v_accvgpr_write acc0, 0x0 // initC +v_accvgpr_write acc1, 0x0 // initC +v_accvgpr_write acc2, 0x0 // initC +v_accvgpr_write acc3, 0x0 // initC +v_accvgpr_write acc4, 0x0 // initC +v_accvgpr_write acc5, 0x0 // initC +v_accvgpr_write acc6, 0x0 // initC +v_accvgpr_write acc7, 0x0 // initC +v_accvgpr_write acc8, 0x0 // initC +v_accvgpr_write acc9, 0x0 // initC +v_accvgpr_write acc10, 0x0 // initC +v_accvgpr_write acc11, 0x0 // initC +v_accvgpr_write acc12, 0x0 // initC +v_accvgpr_write acc13, 0x0 // initC +v_accvgpr_write acc14, 0x0 // initC +v_accvgpr_write acc15, 0x0 // initC +v_accvgpr_write acc16, 0x0 // initC +v_accvgpr_write acc17, 0x0 // initC +v_accvgpr_write acc18, 0x0 // initC +v_accvgpr_write acc19, 0x0 // initC +v_accvgpr_write acc20, 0x0 // initC +v_accvgpr_write acc21, 0x0 // initC +v_accvgpr_write acc22, 0x0 // initC +v_accvgpr_write acc23, 0x0 // initC +v_accvgpr_write acc24, 0x0 // initC +v_accvgpr_write acc25, 0x0 // initC +v_accvgpr_write acc26, 0x0 // initC +v_accvgpr_write acc27, 0x0 // initC +v_accvgpr_write acc28, 0x0 // initC +v_accvgpr_write acc29, 0x0 // initC +v_accvgpr_write acc30, 0x0 // initC +v_accvgpr_write acc31, 0x0 // initC +v_accvgpr_write acc32, 0x0 // initC +v_accvgpr_write acc33, 0x0 // initC +v_accvgpr_write acc34, 0x0 // initC +v_accvgpr_write acc35, 0x0 // initC +v_accvgpr_write acc36, 0x0 // initC +v_accvgpr_write acc37, 0x0 // initC +v_accvgpr_write acc38, 0x0 // initC +v_accvgpr_write acc39, 0x0 // initC +v_accvgpr_write acc40, 0x0 // initC +v_accvgpr_write acc41, 0x0 // initC +v_accvgpr_write acc42, 0x0 // initC +v_accvgpr_write acc43, 0x0 // initC +v_accvgpr_write acc44, 0x0 // initC +v_accvgpr_write acc45, 0x0 // initC +v_accvgpr_write acc46, 0x0 // initC +v_accvgpr_write acc47, 0x0 // initC +v_accvgpr_write acc48, 0x0 // initC +v_accvgpr_write acc49, 0x0 // initC +v_accvgpr_write acc50, 0x0 // initC +v_accvgpr_write acc51, 0x0 // initC +v_accvgpr_write acc52, 0x0 // initC +v_accvgpr_write acc53, 0x0 // initC +v_accvgpr_write acc54, 0x0 // initC +v_accvgpr_write acc55, 0x0 // initC +v_accvgpr_write acc56, 0x0 // initC +v_accvgpr_write acc57, 0x0 // initC +v_accvgpr_write acc58, 0x0 // initC +v_accvgpr_write acc59, 0x0 // initC +v_accvgpr_write acc60, 0x0 // initC +v_accvgpr_write acc61, 0x0 // initC +v_accvgpr_write acc62, 0x0 // initC +v_accvgpr_write acc63, 0x0 // initC +v_accvgpr_write acc64, 0x0 // initC +v_accvgpr_write acc65, 0x0 // initC +v_accvgpr_write acc66, 0x0 // initC +v_accvgpr_write acc67, 0x0 // initC +v_accvgpr_write acc68, 0x0 // initC +v_accvgpr_write acc69, 0x0 // initC +v_accvgpr_write acc70, 0x0 // initC +v_accvgpr_write acc71, 0x0 // initC +v_accvgpr_write acc72, 0x0 // initC +v_accvgpr_write acc73, 0x0 // initC +v_accvgpr_write acc74, 0x0 // initC +v_accvgpr_write acc75, 0x0 // initC +v_accvgpr_write acc76, 0x0 // initC +v_accvgpr_write acc77, 0x0 // initC +v_accvgpr_write acc78, 0x0 // initC +v_accvgpr_write acc79, 0x0 // initC +v_accvgpr_write acc80, 0x0 // initC +v_accvgpr_write acc81, 0x0 // initC +v_accvgpr_write acc82, 0x0 // initC +v_accvgpr_write acc83, 0x0 // initC +v_accvgpr_write acc84, 0x0 // initC +v_accvgpr_write acc85, 0x0 // initC +v_accvgpr_write acc86, 0x0 // initC +v_accvgpr_write acc87, 0x0 // initC +v_accvgpr_write acc88, 0x0 // initC +v_accvgpr_write acc89, 0x0 // initC +v_accvgpr_write acc90, 0x0 // initC +v_accvgpr_write acc91, 0x0 // initC +v_accvgpr_write acc92, 0x0 // initC +v_accvgpr_write acc93, 0x0 // initC +v_accvgpr_write acc94, 0x0 // initC +v_accvgpr_write acc95, 0x0 // initC +v_accvgpr_write acc96, 0x0 // initC +v_accvgpr_write acc97, 0x0 // initC +v_accvgpr_write acc98, 0x0 // initC +v_accvgpr_write acc99, 0x0 // initC +v_accvgpr_write acc100, 0x0 // initC +v_accvgpr_write acc101, 0x0 // initC +v_accvgpr_write acc102, 0x0 // initC +v_accvgpr_write acc103, 0x0 // initC +v_accvgpr_write acc104, 0x0 // initC +v_accvgpr_write acc105, 0x0 // initC +v_accvgpr_write acc106, 0x0 // initC +v_accvgpr_write acc107, 0x0 // initC +v_accvgpr_write acc108, 0x0 // initC +v_accvgpr_write acc109, 0x0 // initC +v_accvgpr_write acc110, 0x0 // initC +v_accvgpr_write acc111, 0x0 // initC +v_accvgpr_write acc112, 0x0 // initC +v_accvgpr_write acc113, 0x0 // initC +v_accvgpr_write acc114, 0x0 // initC +v_accvgpr_write acc115, 0x0 // initC +v_accvgpr_write acc116, 0x0 // initC +v_accvgpr_write acc117, 0x0 // initC +v_accvgpr_write acc118, 0x0 // initC +v_accvgpr_write acc119, 0x0 // initC +v_accvgpr_write acc120, 0x0 // initC +v_accvgpr_write acc121, 0x0 // initC +v_accvgpr_write acc122, 0x0 // initC +v_accvgpr_write acc123, 0x0 // initC +v_accvgpr_write acc124, 0x0 // initC +v_accvgpr_write acc125, 0x0 // initC +v_accvgpr_write acc126, 0x0 // initC +v_accvgpr_write acc127, 0x0 // initC +v_accvgpr_write acc128, 0x0 // initC +v_accvgpr_write acc129, 0x0 // initC +v_accvgpr_write acc130, 0x0 // initC +v_accvgpr_write acc131, 0x0 // initC +v_accvgpr_write acc132, 0x0 // initC +v_accvgpr_write acc133, 0x0 // initC +v_accvgpr_write acc134, 0x0 // initC +v_accvgpr_write acc135, 0x0 // initC +v_accvgpr_write acc136, 0x0 // initC +v_accvgpr_write acc137, 0x0 // initC +v_accvgpr_write acc138, 0x0 // initC +v_accvgpr_write acc139, 0x0 // initC +v_accvgpr_write acc140, 0x0 // initC +v_accvgpr_write acc141, 0x0 // initC +v_accvgpr_write acc142, 0x0 // initC +v_accvgpr_write acc143, 0x0 // initC +v_accvgpr_write acc144, 0x0 // initC +v_accvgpr_write acc145, 0x0 // initC +v_accvgpr_write acc146, 0x0 // initC +v_accvgpr_write acc147, 0x0 // initC +v_accvgpr_write acc148, 0x0 // initC +v_accvgpr_write acc149, 0x0 // initC +v_accvgpr_write acc150, 0x0 // initC +v_accvgpr_write acc151, 0x0 // initC +v_accvgpr_write acc152, 0x0 // initC +v_accvgpr_write acc153, 0x0 // initC +v_accvgpr_write acc154, 0x0 // initC +v_accvgpr_write acc155, 0x0 // initC +v_accvgpr_write acc156, 0x0 // initC +v_accvgpr_write acc157, 0x0 // initC +v_accvgpr_write acc158, 0x0 // initC +v_accvgpr_write acc159, 0x0 // initC +v_accvgpr_write acc160, 0x0 // initC +v_accvgpr_write acc161, 0x0 // initC +v_accvgpr_write acc162, 0x0 // initC +v_accvgpr_write acc163, 0x0 // initC +v_accvgpr_write acc164, 0x0 // initC +v_accvgpr_write acc165, 0x0 // initC +v_accvgpr_write acc166, 0x0 // initC +v_accvgpr_write acc167, 0x0 // initC +v_accvgpr_write acc168, 0x0 // initC +v_accvgpr_write acc169, 0x0 // initC +v_accvgpr_write acc170, 0x0 // initC +v_accvgpr_write acc171, 0x0 // initC +v_accvgpr_write acc172, 0x0 // initC +v_accvgpr_write acc173, 0x0 // initC +v_accvgpr_write acc174, 0x0 // initC +v_accvgpr_write acc175, 0x0 // initC +v_accvgpr_write acc176, 0x0 // initC +v_accvgpr_write acc177, 0x0 // initC +v_accvgpr_write acc178, 0x0 // initC +v_accvgpr_write acc179, 0x0 // initC +v_accvgpr_write acc180, 0x0 // initC +v_accvgpr_write acc181, 0x0 // initC +v_accvgpr_write acc182, 0x0 // initC +v_accvgpr_write acc183, 0x0 // initC +v_accvgpr_write acc184, 0x0 // initC +v_accvgpr_write acc185, 0x0 // initC +v_accvgpr_write acc186, 0x0 // initC +v_accvgpr_write acc187, 0x0 // initC +v_accvgpr_write acc188, 0x0 // initC +v_accvgpr_write acc189, 0x0 // initC +v_accvgpr_write acc190, 0x0 // initC +v_accvgpr_write acc191, 0x0 // initC +v_accvgpr_write acc192, 0x0 // initC +v_accvgpr_write acc193, 0x0 // initC +v_accvgpr_write acc194, 0x0 // initC +v_accvgpr_write acc195, 0x0 // initC +v_accvgpr_write acc196, 0x0 // initC +v_accvgpr_write acc197, 0x0 // initC +v_accvgpr_write acc198, 0x0 // initC +v_accvgpr_write acc199, 0x0 // initC +v_accvgpr_write acc200, 0x0 // initC +v_accvgpr_write acc201, 0x0 // initC +v_accvgpr_write acc202, 0x0 // initC +v_accvgpr_write acc203, 0x0 // initC +v_accvgpr_write acc204, 0x0 // initC +v_accvgpr_write acc205, 0x0 // initC +v_accvgpr_write acc206, 0x0 // initC +v_accvgpr_write acc207, 0x0 // initC +v_accvgpr_write acc208, 0x0 // initC +v_accvgpr_write acc209, 0x0 // initC +v_accvgpr_write acc210, 0x0 // initC +v_accvgpr_write acc211, 0x0 // initC +v_accvgpr_write acc212, 0x0 // initC +v_accvgpr_write acc213, 0x0 // initC +v_accvgpr_write acc214, 0x0 // initC +v_accvgpr_write acc215, 0x0 // initC +v_accvgpr_write acc216, 0x0 // initC +v_accvgpr_write acc217, 0x0 // initC +v_accvgpr_write acc218, 0x0 // initC +v_accvgpr_write acc219, 0x0 // initC +v_accvgpr_write acc220, 0x0 // initC +v_accvgpr_write acc221, 0x0 // initC +v_accvgpr_write acc222, 0x0 // initC +v_accvgpr_write acc223, 0x0 // initC +v_accvgpr_write acc224, 0x0 // initC +v_accvgpr_write acc225, 0x0 // initC +v_accvgpr_write acc226, 0x0 // initC +v_accvgpr_write acc227, 0x0 // initC +v_accvgpr_write acc228, 0x0 // initC +v_accvgpr_write acc229, 0x0 // initC +v_accvgpr_write acc230, 0x0 // initC +v_accvgpr_write acc231, 0x0 // initC +v_accvgpr_write acc232, 0x0 // initC +v_accvgpr_write acc233, 0x0 // initC +v_accvgpr_write acc234, 0x0 // initC +v_accvgpr_write acc235, 0x0 // initC +v_accvgpr_write acc236, 0x0 // initC +v_accvgpr_write acc237, 0x0 // initC +v_accvgpr_write acc238, 0x0 // initC +v_accvgpr_write acc239, 0x0 // initC +v_accvgpr_write acc240, 0x0 // initC +v_accvgpr_write acc241, 0x0 // initC +v_accvgpr_write acc242, 0x0 // initC +v_accvgpr_write acc243, 0x0 // initC +v_accvgpr_write acc244, 0x0 // initC +v_accvgpr_write acc245, 0x0 // initC +v_accvgpr_write acc246, 0x0 // initC +v_accvgpr_write acc247, 0x0 // initC +v_accvgpr_write acc248, 0x0 // initC +v_accvgpr_write acc249, 0x0 // initC +v_accvgpr_write acc250, 0x0 // initC +v_accvgpr_write acc251, 0x0 // initC +v_accvgpr_write acc252, 0x0 // initC +v_accvgpr_write acc253, 0x0 // initC +v_accvgpr_write acc254, 0x0 // initC +v_accvgpr_write acc255, 0x0 // initC +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? + +/* after InitC, skip to end of prefetch last iter if numIter==0 */ +s_cbranch_scc0 label_NoBranch_5L94Q2PNAOS4OEC5_0 // Only branch on scc1 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_5L94Q2PNAOS4OEC5_0: +s_waitcnt vmcnt(8) +/* local write a */ + +/* local write b */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 + +/* local write swap a */ + +/* local write swap b */ +s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // PGR=2 but only 1 loop +s_cbranch_scc1 label_skipPGR2_0 // PGR=2 but only 1 loop +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 + +label_skipPGR2_0: +s_waitcnt lgkmcnt(0) // 0prefetch wait for local write +// Skip force waitcnt0 +s_barrier + +/* local read prefetch a */ + +/* local read prefetch b */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + +/* local read inc a */ +/* N/A, lro->32 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + +/* local read inc b */ +/* N/A, lro->32 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 + +/******************************************/ +/* Unrolled Loop(s) - Begin */ +/******************************************/ +label_openLoopL: +s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // LoopCounterL < EndCounter +s_cbranch_scc1 label_LoopEndL_odd_NoLoadLoop +s_cmp_le_u32 s[sgprLoopCounterL], 0x2 // LoopCounterL < EndCounter +s_cbranch_scc1 label_LoopEndL_even // do not enter LoopL +label_LoopBeginL: + +/******************************************/ +/* Unrolled Loop 1/2 - Begin */ +/******************************************/ + +s_waitcnt vmcnt(8) + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + +/* closeLoop loopL finalLoop=1 tailLoop=0 */ +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL +s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==1 +s_cbranch_scc1 label_LoopEndL_odd // to End + +/******************************************/ +/* Unrolled Loop 2/2 - Begin */ +/******************************************/ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +s_waitcnt vmcnt(8) + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + +/* closeLoop loopL finalLoop=1 tailLoop=0 */ +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL +s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==2 +s_cbranch_scc0 label_LoopBeginL // restart LoopL +label_LoopEndL_even: + +/* Before NLL: Check VGPR.checkin for INT8 LW */ + +/******************************************/ +/* Ord. NoGlobalLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(8) +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(10) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(9) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(8) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/******************************************/ +/* Ord. NoLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(0) + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +label_Summation_End_OptNLL: +s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 +s_cbranch_scc1 label_NoBranch_PAGZHD9H2DI57HHE_0 // Only branch on scc0 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_PAGZHD9H2DI57HHE_0: + +s_cmp_eq_u32 s[sgprAlpha], 1.0 // Alpha == 1.0 ? +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_XQG82FMXJOJL8OIW_0: + +s_and_b32 s80, 255, s[sgprSizeI] // s80 = s[sgprSizeI] % 256 +s_add_u32 s81, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s81 // wg0 >= nwg0-1 ? +s_cselect_b32 s80, s80, 0 // set rMT0 +s_cmpk_gt_u32 s80, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_XQ75AI1RJ5F179IN_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_XQ75AI1RJ5F179IN_0: +s_and_b32 s80, 255, s[sgprSizeJ] // s80 = s[sgprSizeJ] % 256 +s_add_u32 s81, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s81 // wg1 >= nwg1-1 +s_cselect_b32 s80, s80, 0 // set rMT1 +s_cmpk_gt_u32 s80, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_GFF6GDU2NIUMLQ8E_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_GFF6GDU2NIUMLQ8E_0: + +s_and_b32 s81, 63, s[sgprSizesSum+0] // s81 = s[sgprSizesSum+0] % 64 +s_cmp_eq_u32 s81, 0x0 // numIterL == 0 +s_cbranch_scc1 label_NoBranch_VEIX0X0UO5PRLIZN_0 // Only branch on scc0 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_VEIX0X0UO5PRLIZN_0: + +/* endSummation: add vgpr [0...230) to pool */ +/* load store sgprs */ +.set sgprAddressScaleAlphaVec, 48 +.set sgprAddressBias, 50 +.set sgprBiasType, 52 +.set sgprBiasStride, 53 +.set sgpractivationAlpha, 54 +.set sgpractivationBeta, 55 +.set sgprActivationType, 56 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalEpilogueStruct // branch if ArgType == 2 +s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 +s_branch label_LoadExternalEpilogueStructEnd +label_LoadExternalEpilogueStruct: +s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 +s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 +s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 +label_LoadExternalEpilogueStructEnd: +.set sgprSrdScaleAlphaVec, 32 +.set sgprSrdBias, 40 + +/* Mapping of Acc register -> C Vgpr register */ +/* computeStoreVgprs */ +v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 +v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 +v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 +v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 +v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 +v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output +v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) +v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 +v_and_b32 v0, 3, v4 // v0 = v4 % 4 +v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 +v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 +v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 + +/******************************************/ +/* Global Write Elements */ +/******************************************/ +s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. +s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? +s_cbranch_scc0 label_ScaleAlphaVecAddrValid // branch if s[AddressScaleAlphaVec] != 0 +s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 +s_branch label_ScaleAlphaVecAddrValid_End +label_ScaleAlphaVecAddrValid: +s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] +label_ScaleAlphaVecAddrValid_End: + +s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE +s_add_u32 s8, s[sgprWorkGroup2], 0x1 +s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) +s_cmp_eq_u32 s8, 0x0 // bias stride = 0? +s_cselect_b32 s8, s[sgprSizeI], s8 +s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? +s_cbranch_scc0 label_BiasAddrValid // branch if s[AddressBias] != 0 +s_mov_b32 s[sgprSrdBias+2], 0 +s_branch label_BiasAddrValid_End +label_BiasAddrValid: +s_mov_b32 s[sgprSrdBias+2], s8 +label_BiasAddrValid_End: + +label_Load_Biasf32_0: +s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 +s_cbranch_scc1 label_Load_Biasbf16_0 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE +buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End // Branch to load bias end +label_Load_Biasbf16_0: +s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 +s_cbranch_scc1 label_Load_Bias_End // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE +buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End // Branch to load bias end +label_Load_Bias_End: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4 // Branch if true +label_To_Activation_None_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Abs_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Clippedrelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Gelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Leakyrelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Relu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Sigmoid_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Tanh_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Geluscaling_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Silu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_ActivationSetPCAddrEnd: +label_GW_B0_E0: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End // jump to end +label_GW_End: + +s_endpgm // Kernel End + +label_LoopEndL_odd: +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/* Before NLL: Check VGPR.checkin for INT8 LW */ + +/******************************************/ +/* Ord. NoGlobalLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(8) +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(10) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(9) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(8) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 + +s_branch label_LoopEndL_odd_NoLoadLoop + +label_LoopEndL_odd_NoLoadLoop: + +/******************************************/ +/* Ord. NoLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(0) + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +s_branch label_Summation_End_OptNLL + +label_PrefetchGlobalLastIterEnd: +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/******************************************/ +/* Tail Loop */ +/******************************************/ + +/* Tail: add ValuA/B vgpr buffer [0...160) to pool */ + +/* local write reset offsets a */ + +/* local write reset offsets b */ + +// numIterL = LOCAL_SPLITU * min(sizeL % LOCAL_DEPTHU, DEPTHU / LOCAL_SPLITU) +s_and_b32 s[sgprLoopCounterL], 63, s[sgprSizesSum+0] // s[sgprLoopCounterL] = s[sgprSizesSum+0] % 64 +s_cmp_lg_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx == numIterPerWgRemainder +s_cmov_b32 s[sgprLoopCounterL], 0x0 // numIter=0 if gsuSimIdx!=remainder +s_cmp_eq_u32 s[sgprLoopCounterL], 0x0 // numIterL == 0 +s_mov_b32 s[sgprOrigLoopCounter], 0 // repurpose to count each localRead increment +s_cbranch_scc1 label_SkipTailLoopL // skip to end of tail loop b/c numIter==0 + +/* remove stagger offsets for tail loop */ +s_sub_i32 s80, 3, s[sgprStaggerUIter] +s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_mul_i32 s80, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_sub_u32 s80, s80, s[sgprWrapUA] // S - WrapU +s_subb_u32 s81, s81, s[sgprWrapUA+1] // S - WrapU +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +s_sub_i32 s80, 3, s[sgprStaggerUIter] +s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_mul_i32 s80, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_sub_u32 s80, s80, s[sgprWrapUB] // S - WrapU +s_subb_u32 s81, s81, s[sgprWrapUB+1] // S - WrapU +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* Recalc global read offsets */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0 15, v1 // 1. M offset: mIdx = wtid % MI_M(16) +v_mul_lo_u32 v0, s[sgprStrideA0I], v0 // 1. M offset: mOffset = mIdx * mStride(k) +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) +v_mul_lo_u32 v1, s[sgprStrideA0I], v1 // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) +v_lshlrev_b32 v1, 0x6, v1 // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) +v_add_u32 v[vgprGlobalReadOffsetA], v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +v_add_u32 v[vgprGlobalReadOffsetA] 0x8 v[vgprGlobalReadOffsetA] // add prepad for pointer shift +v_lshlrev_b32 v[vgprGlobalReadOffsetA] 0x1 v[vgprGlobalReadOffsetA] // offset *= bytes/element + +s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 16 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] + +/* Update M0 for DTLDS */ + +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+0], v[vgprValuA_X0_I0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+1], v[vgprValuA_X0_I0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+8], v[vgprValuA_X0_I0+8], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+9], v[vgprValuA_X0_I0+9], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 64 +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:64 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:66 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+0], v[vgprValuA_X2_I0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:68 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:70 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+1], v[vgprValuA_X2_I0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+8], v[vgprValuA_X2_I0+8], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+9], v[vgprValuA_X2_I0+9], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+2], v[vgprValuA_X0_I0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+3], v[vgprValuA_X0_I0+3], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 64 +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+10], v[vgprValuA_X0_I0+10], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+11], v[vgprValuA_X0_I0+11], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+4], 64 +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+2], v[vgprValuA_X2_I0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+3], v[vgprValuA_X2_I0+3], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+10], v[vgprValuA_X2_I0+10], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+11], v[vgprValuA_X2_I0+11], v0 // HasEccHalf: pack +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+4], v[vgprValuA_X0_I0+4], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+5], v[vgprValuA_X0_I0+5], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 64 +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+12], v[vgprValuA_X0_I0+12], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+13], v[vgprValuA_X0_I0+13], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+5], 64 +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+4], v[vgprValuA_X2_I0+4], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+5], v[vgprValuA_X2_I0+5], v0 // HasEccHalf: pack +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+12], v[vgprValuA_X2_I0+12], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+13], v[vgprValuA_X2_I0+13], v0 // HasEccHalf: pack +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+6], v[vgprValuA_X0_I0+6], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+7], v[vgprValuA_X0_I0+7], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 64 +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+14], v[vgprValuA_X0_I0+14], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+15], v[vgprValuA_X0_I0+15], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+6], 64 +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+6], v[vgprValuA_X2_I0+6], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+7], v[vgprValuA_X2_I0+7], v0 // HasEccHalf: pack +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+14], v[vgprValuA_X2_I0+14], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+15], v[vgprValuA_X2_I0+15], v0 // HasEccHalf: pack + +/* Update M0 for DTLDS */ + +/* global read B */ +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+0+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+0], v[vgprG2LB+0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+0+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+1], v[vgprG2LB+0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+0+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:8 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+2], v[vgprG2LB+0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:12 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+3], v[vgprG2LB+0+3], v0 // HasEccHalf: pack +/* g2l=4, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+4+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // load one buffer value +/* g2l=4, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+0], v[vgprG2LB+4+0], v0 // HasEccHalf: pack +/* g2l=4, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+4+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:4 // load one buffer value +/* g2l=4, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+1], v[vgprG2LB+4+1], v0 // HasEccHalf: pack +/* g2l=4, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+4+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:8 // load one buffer value +/* g2l=4, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+2], v[vgprG2LB+4+2], v0 // HasEccHalf: pack +/* g2l=4, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:12 // load one buffer value +/* g2l=4, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+3], v[vgprG2LB+4+3], v0 // HasEccHalf: pack +/* g2l=8, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+8+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // load one buffer value +/* g2l=8, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+0], v[vgprG2LB+8+0], v0 // HasEccHalf: pack +/* g2l=8, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+8+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:4 // load one buffer value +/* g2l=8, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+1], v[vgprG2LB+8+1], v0 // HasEccHalf: pack +/* g2l=8, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+8+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:8 // load one buffer value +/* g2l=8, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+2], v[vgprG2LB+8+2], v0 // HasEccHalf: pack +/* g2l=8, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:12 // load one buffer value +/* g2l=8, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+3], v[vgprG2LB+8+3], v0 // HasEccHalf: pack +/* g2l=12, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+12+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // load one buffer value +/* g2l=12, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+0], v[vgprG2LB+12+0], v0 // HasEccHalf: pack +/* g2l=12, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+12+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:4 // load one buffer value +/* g2l=12, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+1], v[vgprG2LB+12+1], v0 // HasEccHalf: pack +/* g2l=12, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+12+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:8 // load one buffer value +/* g2l=12, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+2], v[vgprG2LB+12+2], v0 // HasEccHalf: pack +/* g2l=12, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:12 // load one buffer value +/* g2l=12, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+3], v[vgprG2LB+12+3], v0 // HasEccHalf: pack +/* g2l=16, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+16+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // load one buffer value +/* g2l=16, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+0], v[vgprG2LB+16+0], v0 // HasEccHalf: pack +/* g2l=16, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+16+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:4 // load one buffer value +/* g2l=16, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+1], v[vgprG2LB+16+1], v0 // HasEccHalf: pack +/* g2l=16, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+16+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:8 // load one buffer value +/* g2l=16, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+2], v[vgprG2LB+16+2], v0 // HasEccHalf: pack +/* g2l=16, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:12 // load one buffer value +/* g2l=16, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+3], v[vgprG2LB+16+3], v0 // HasEccHalf: pack +/* g2l=20, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+20+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // load one buffer value +/* g2l=20, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+0], v[vgprG2LB+20+0], v0 // HasEccHalf: pack +/* g2l=20, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+20+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:4 // load one buffer value +/* g2l=20, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+1], v[vgprG2LB+20+1], v0 // HasEccHalf: pack +/* g2l=20, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+20+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:8 // load one buffer value +/* g2l=20, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+2], v[vgprG2LB+20+2], v0 // HasEccHalf: pack +/* g2l=20, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:12 // load one buffer value +/* g2l=20, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+3], v[vgprG2LB+20+3], v0 // HasEccHalf: pack +/* g2l=24, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+24+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // load one buffer value +/* g2l=24, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+0], v[vgprG2LB+24+0], v0 // HasEccHalf: pack +/* g2l=24, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+24+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:4 // load one buffer value +/* g2l=24, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+1], v[vgprG2LB+24+1], v0 // HasEccHalf: pack +/* g2l=24, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+24+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:8 // load one buffer value +/* g2l=24, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+2], v[vgprG2LB+24+2], v0 // HasEccHalf: pack +/* g2l=24, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:12 // load one buffer value +/* g2l=24, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+3], v[vgprG2LB+24+3], v0 // HasEccHalf: pack +/* g2l=28, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+28+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // load one buffer value +/* g2l=28, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+0], v[vgprG2LB+28+0], v0 // HasEccHalf: pack +/* g2l=28, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+28+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:4 // load one buffer value +/* g2l=28, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+1], v[vgprG2LB+28+1], v0 // HasEccHalf: pack +/* g2l=28, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+28+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:8 // load one buffer value +/* g2l=28, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+2], v[vgprG2LB+28+2], v0 // HasEccHalf: pack +/* g2l=28, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:12 // load one buffer value +/* g2l=28, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+3], v[vgprG2LB+28+3], v0 // HasEccHalf: pack +s_waitcnt vmcnt(0) // 2wait for global read +// Skip force waitcnt0 +s_barrier + +/* local write a */ + +/* local write b */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 + +/* Recalc local read offsets */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x2, v2 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s8, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 +/* N/A */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s8, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 +s_waitcnt lgkmcnt(0) // 5wait for local write +// Skip force waitcnt0 +s_barrier + +/* local read reset offsets a */ + +/* local read reset offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ + +/* tail loop: macs */ +label_TailLoopBeginL: + +/* Tail: remove ValuA/B vgpr buffer [0...160) from pool */ + +/* Tail: add address/G2L vgpr [160...230) to pool */ + +/* local read a */ + +/* local read b */ +ds_read_b64 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+1], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+2+1], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+1], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+6+1], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+1], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+10+1], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+1], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+14+1], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+1], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+18:vgprValuB_X0_I0+18+1], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+1], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+22:vgprValuB_X0_I0+22+1], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+1], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+26:vgprValuB_X0_I0+26+1], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+1], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+30:vgprValuB_X0_I0+30+1], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + +/* local read inc a */ +s_mov_b32 s8, 0x20 // inc + +/* local read inc b */ +s_mov_b32 s8, 0x20 // inc +v_add_co_u32 v[vgprLocalReadAddrB], vcc, s8, v[vgprLocalReadAddrB] // lrB += 32 (bpeDS) +s_waitcnt lgkmcnt(0) // 4wait for local read +v_and_b32 v160, 63, v[vgprSerial] // v160 = v[vgprSerial] % 64 +v_lshrrev_b32 v160, 4, v160 // v160 = v160 / 16 +v_lshlrev_b32 v160, 0x2, v160 // v160 = v160 * 4 +v_cmp_ge_i32 s[80:81], v160, s[sgprLoopCounterL] // check K index >= Size L +v_cndmask_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+0+1], v[vgprValuA_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+1], v[vgprValuA_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+4+1], v[vgprValuA_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+6+1], v[vgprValuA_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+0], v[vgprValuB_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+0], v[vgprValuB_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+0], v[vgprValuB_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+0], v[vgprValuB_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+0], v[vgprValuB_X0_I0+8+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+0], v[vgprValuB_X0_I0+10+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+0], v[vgprValuB_X0_I0+12+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+0], v[vgprValuB_X0_I0+14+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+16+0], v[vgprValuB_X0_I0+16+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+18+0], v[vgprValuB_X0_I0+18+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+20+0], v[vgprValuB_X0_I0+20+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+22+0], v[vgprValuB_X0_I0+22+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+24+0], v[vgprValuB_X0_I0+24+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+26+0], v[vgprValuB_X0_I0+26+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+28+0], v[vgprValuB_X0_I0+28+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+30+0], v[vgprValuB_X0_I0+30+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+1], v[vgprValuB_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+1], v[vgprValuB_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+1], v[vgprValuB_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+1], v[vgprValuB_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+1], v[vgprValuB_X0_I0+8+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+1], v[vgprValuB_X0_I0+10+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+1], v[vgprValuB_X0_I0+12+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+1], v[vgprValuB_X0_I0+14+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+16+1], v[vgprValuB_X0_I0+16+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+18+1], v[vgprValuB_X0_I0+18+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+20+1], v[vgprValuB_X0_I0+20+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+22+1], v[vgprValuB_X0_I0+22+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+24+1], v[vgprValuB_X0_I0+24+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+26+1], v[vgprValuB_X0_I0+26+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+28+1], v[vgprValuB_X0_I0+28+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+30+1], v[vgprValuB_X0_I0+30+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_sub_u32 v160, s[sgprLoopCounterL], v160 // get distance between size and k index +v_cmp_lt_i32 s[80:81], v160, 4 // set partial 0 if distance less than input per thread +s_and_b32 s82, s[sgprLoopCounterL], 3 // get inputs for edge thread +s_sub_u32 s82, 4, s82 // use shift to fill 0 for outside element +s_lshl_b32 s82, s82, 4 // use shift to fill 0 for outside element +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+0], v[vgprValuA_X0_I0+0+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+0], v[vgprValuA_X0_I0+2+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+0], v[vgprValuA_X0_I0+4+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+0], v[vgprValuA_X0_I0+6+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+0], v[vgprValuB_X0_I0+0+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+1], v[vgprValuB_X0_I0+0+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+0], v[vgprValuB_X0_I0+2+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+1], v[vgprValuB_X0_I0+2+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+0], v[vgprValuB_X0_I0+4+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+1], v[vgprValuB_X0_I0+4+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+0], v[vgprValuB_X0_I0+6+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+1], v[vgprValuB_X0_I0+6+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+0], v[vgprValuB_X0_I0+8+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+1], v[vgprValuB_X0_I0+8+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+0], v[vgprValuB_X0_I0+10+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+1], v[vgprValuB_X0_I0+10+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+0], v[vgprValuB_X0_I0+12+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+1], v[vgprValuB_X0_I0+12+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+0], v[vgprValuB_X0_I0+14+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+1], v[vgprValuB_X0_I0+14+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+0], v[vgprValuB_X0_I0+16+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+1], v[vgprValuB_X0_I0+16+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+0], v[vgprValuB_X0_I0+18+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+1], v[vgprValuB_X0_I0+18+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+0], v[vgprValuB_X0_I0+20+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+1], v[vgprValuB_X0_I0+20+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+0], v[vgprValuB_X0_I0+22+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+1], v[vgprValuB_X0_I0+22+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+0], v[vgprValuB_X0_I0+24+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+1], v[vgprValuB_X0_I0+24+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+0], v[vgprValuB_X0_I0+26+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+1], v[vgprValuB_X0_I0+26+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+0], v[vgprValuB_X0_I0+28+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+1], v[vgprValuB_X0_I0+28+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+0], v[vgprValuB_X0_I0+30+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+1], v[vgprValuB_X0_I0+30+0+0+1], v163, s[80:81] +s_nop 1 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[252:255] // left value = acc[252+0:255+0] + +v_mov_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+8+0] +v_mov_b32 v[vgprValuA_X0_I0+1+0], v[vgprValuA_X0_I0+9+0] +v_mov_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+10+0] +v_mov_b32 v[vgprValuA_X0_I0+3+0], v[vgprValuA_X0_I0+11+0] +v_mov_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+12+0] +v_mov_b32 v[vgprValuA_X0_I0+5+0], v[vgprValuA_X0_I0+13+0] +v_mov_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+14+0] +v_mov_b32 v[vgprValuA_X0_I0+7+0], v[vgprValuA_X0_I0+15+0] +v_mov_b32 v[vgprValuA_X0_I0+8+0], v[vgprValuA_X2_I0+0+0] +v_mov_b32 v[vgprValuA_X0_I0+9+0], v[vgprValuA_X2_I0+1+0] +v_mov_b32 v[vgprValuA_X0_I0+10+0], v[vgprValuA_X2_I0+2+0] +v_mov_b32 v[vgprValuA_X0_I0+11+0], v[vgprValuA_X2_I0+3+0] +v_mov_b32 v[vgprValuA_X0_I0+12+0], v[vgprValuA_X2_I0+4+0] +v_mov_b32 v[vgprValuA_X0_I0+13+0], v[vgprValuA_X2_I0+5+0] +v_mov_b32 v[vgprValuA_X0_I0+14+0], v[vgprValuA_X2_I0+6+0] +v_mov_b32 v[vgprValuA_X0_I0+15+0], v[vgprValuA_X2_I0+7+0] +v_mov_b32 v[vgprValuA_X2_I0+0+0], v[vgprValuA_X2_I0+8+0] +v_mov_b32 v[vgprValuA_X2_I0+1+0], v[vgprValuA_X2_I0+9+0] +v_mov_b32 v[vgprValuA_X2_I0+2+0], v[vgprValuA_X2_I0+10+0] +v_mov_b32 v[vgprValuA_X2_I0+3+0], v[vgprValuA_X2_I0+11+0] +v_mov_b32 v[vgprValuA_X2_I0+4+0], v[vgprValuA_X2_I0+12+0] +v_mov_b32 v[vgprValuA_X2_I0+5+0], v[vgprValuA_X2_I0+13+0] +v_mov_b32 v[vgprValuA_X2_I0+6+0], v[vgprValuA_X2_I0+14+0] +v_mov_b32 v[vgprValuA_X2_I0+7+0], v[vgprValuA_X2_I0+15+0] + +/* closeLoop loopL finalLoop=1 tailLoop=1 */ +s_sub_i32 s[sgprLoopCounterL], s[sgprLoopCounterL], 0x10 // dec counterL (tailLoop) +s_add_u32 s[sgprOrigLoopCounter], s[sgprOrigLoopCounter], 0x10 // inc counterL +s_cmp_le_i32 s[sgprLoopCounterL], 0x0 // counterL<=0 +s_cbranch_scc0 label_TailLoopBeginL // restart LoopL +label_TailLoopEndL: +label_SkipTailLoopL: + +/* Tail: remove address/G2L [160...230) from pool */ +label_Summation_End_SB904UR36QKZ73J2_0: +/* endSummation: add vgpr [0...230) to pool */ +.set sgprWGM, UNDEF +.set sgprLoopCounterL, UNDEF +.set sgprOrigLoopCounter, UNDEF +.set sgprAddressA, UNDEF +.set sgprAddressB, UNDEF +.set sgprStridesA, UNDEF +.set sgprStridesB, UNDEF +.set sgprStaggerUIter, UNDEF +.set sgprSrdA, UNDEF +.set sgprSrdB, UNDEF +.set sgprShadowLimitA, UNDEF +.set sgprShadowLimitB, UNDEF +.set sgprWrapUA, UNDEF +.set sgprWrapUB, UNDEF +.set sgprGlobalReadIncsA, UNDEF +.set sgprGlobalReadIncsB, UNDEF +.set sgprScalarGlobalReadOffsetA, UNDEF +.set sgprScalarGlobalReadOffsetB, UNDEF +/* load store sgprs */ +.set sgprAddressScaleAlphaVec, 48 +.set sgprAddressBias, 50 +.set sgprBiasType, 52 +.set sgprBiasStride, 53 +.set sgpractivationAlpha, 54 +.set sgpractivationBeta, 55 +.set sgprActivationType, 56 +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc0 label_GSU_4 // branch if GSU != 1 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalEpilogueStruct_1 // branch if ArgType == 2 +s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 +s_branch label_LoadExternalEpilogueStructEnd_1 +label_LoadExternalEpilogueStruct_1: +s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 +s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 +s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 +label_LoadExternalEpilogueStructEnd_1: +label_GSU_4: +.set sgprSrdScaleAlphaVec, 32 +.set sgprSrdBias, 40 + +/* Mapping of Acc register -> C Vgpr register */ + +/* not-LocalSplitU: global write indices */ +/* computeStoreVgprs */ +v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 +v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 +v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 +v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 +v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 +v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output +v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) +v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 +v_and_b32 v0, 3, v4 // v0 = v4 % 4 +v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 +v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 +v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 + +/* not-LocalSplitU: global write */ + +/******************************************/ +/* Global Write Elements */ +/******************************************/ +s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_5 // branch if GSU == 1 +s_and_b32 s58, 255, s[sgprSizeI] // s58 = s[sgprSizeI] % 256 +s_add_u32 s59, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s59 // wg0 >= nwg0-1 ? +s_cselect_b32 s58, s58, 0 // set rMT0 +s_cmpk_gt_u32 s58, 0x0 // rMT0 > 0 +s_cbranch_scc1 label_GW_B0_E1_M // jump if edges required +s_and_b32 s58, 255, s[sgprSizeJ] // s58 = s[sgprSizeJ] % 256 +s_add_u32 s59, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s59 // wg1 >= nwg1-1 +s_cselect_b32 s58, s58, 0 // set rMT1 +s_cmpk_gt_u32 s58, 0x0 // rMT1 > 0 +s_cbranch_scc1 label_GW_B0_E1_N // jump if edges required +label_GW_B0_E0_1: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v10, v3, v0, 0x2 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+20], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+21], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+22], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+23], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+24], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+25], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+26], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+27], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+28], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+29], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+31], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+32], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+33], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+34], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+35], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+36], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+37], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+38], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+39], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+40], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+41], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+42], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+43], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+44], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+45], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+46], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+47], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+48], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+49], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+50], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+51], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+52], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+53], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+54], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+55], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+56], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+57], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+58], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+59], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+60], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+61], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+62], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+63], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+64], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+65], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+66], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+67], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+68], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+69], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+70], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+71], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+73], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+74], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+75], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+20], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+21], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+22], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+23], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+24], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+25], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+26], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+27], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+28], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+29], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+31], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+32], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+33], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+34], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+35], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+36], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+37], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+38], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+39], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+40], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+41], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+42], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+43], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+44], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+45], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+46], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+47], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+48], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+49], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+50], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+51], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+52], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+53], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+54], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+55], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+60], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+61], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+62], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+63], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+64], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+65], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+66], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+67], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+68], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+69], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+70], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+71], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+73], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+74], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+75], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+20], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+21], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+22], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+23], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+24], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+25], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+26], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+27], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+32], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+33], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+34], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+35], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+36], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+37], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+38], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+39], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+40], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+41], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+42], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+43], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+44], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+45], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+46], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+47], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+48], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+49], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+50], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+51], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+52], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+53], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+54], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+55], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+56], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+57], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+58], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+59], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+60], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+61], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+62], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+63], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+64], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+65], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+66], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+67], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+68], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+69], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+70], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+71], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+73], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+74], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+75], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+20], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+21], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+22], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+23], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+24], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+25], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+26], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+27], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+28], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+29], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+31], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+32], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+33], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+34], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+35], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+36], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+37], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+38], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+39], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+40], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+41], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+42], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+43], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+44], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+45], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+46], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+47], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+48], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+49], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+50], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+51], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+52], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+53], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+54], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+55], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+56], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+57], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+58], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+59], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+60], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+61], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+62], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+63], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+64], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+65], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+66], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+67], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+73], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+74], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+75], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_B0_E1_N: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+24], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+25], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+26], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+27], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+28], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+29], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+30], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+31], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+32], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+33], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+34], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+35], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+36], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+37], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+38], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+39], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+44], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+45], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+46], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+47], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+48], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+49], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+50], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+51], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+52], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+53], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+54], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+55], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+56], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+57], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+58], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+59], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+64], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+65], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+66], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+67], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+68], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+69], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+70], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+71], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+72], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+73], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+74], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+75], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+76], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+77], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+78], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+79], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+24], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+25], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+26], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+27], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+28], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+29], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+30], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+31], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+32], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+33], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+34], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+35], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+36], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+37], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+38], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+39], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+44], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+45], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+46], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+47], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+48], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+49], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+50], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+51], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+52], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+53], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+54], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+55], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+56], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+57], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+58], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+59], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+64], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+65], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+66], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+67], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+68], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+69], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+70], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+71], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+72], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+73], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+74], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+75], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+76], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+77], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+78], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+79], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+24], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+25], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+26], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+27], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+28], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+29], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+30], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+31], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+32], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+33], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+34], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+35], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+48], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+49], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+50], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+51], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+52], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+53], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+54], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+55], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+56], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+57], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+58], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+59], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+64], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+65], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+66], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+67], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+68], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+69], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+70], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+71], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+72], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+73], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+74], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+75], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+76], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+77], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+78], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+79], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+24], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+25], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+26], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+27], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+28], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+29], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+30], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+31], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+32], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+33], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+34], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+35], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+36], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+37], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+38], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+39], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+44], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+45], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+46], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+47], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+48], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+49], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+50], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+51], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+52], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+53], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+54], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+55], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+56], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+57], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+58], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+59], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+64], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+65], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+66], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+67], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+68], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+69], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+70], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+71], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+72], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+73], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+74], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+75], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+76], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+77], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+78], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+79], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_B0_E1_M: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+15], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+17], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+19], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+21], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+23], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+25], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+27], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+29], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+31], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+33], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+35], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+37], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+39], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+41], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+13], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+15], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+17], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+19], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+21], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+23], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+25], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+27], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+29], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+31], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+33], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+35], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+37], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+39], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+41], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+13], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+15], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+17], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+19], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+21], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+23], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+25], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+27], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+29], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+31], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+33], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+35], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+37], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+39], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+41], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+13], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+15], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+17], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+19], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+21], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+23], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+25], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+27], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+29], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+31], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+33], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+35], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+37], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+39], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+41], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+15], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+17], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+19], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+21], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+23], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+25], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+27], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+29], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+31], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+33], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+35], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+37], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+39], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+41], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+13], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+15], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+17], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+19], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+21], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+23], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+25], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+27], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+29], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+31], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+33], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+35], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+37], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+39], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+41], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+13], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+15], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+17], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+19], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+21], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+23], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+25], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+27], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+29], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+31], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+33], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+35], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+37], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+39], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+41], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+13], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+15], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+17], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+19], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+21], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+23], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+25], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+27], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+29], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+31], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+33], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+35], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+37], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+39], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+41], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+15], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+17], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+19], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+21], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+23], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+25], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+27], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+29], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+31], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+33], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+35], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+37], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+39], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+41], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+13], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+15], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+17], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+19], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+21], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+23], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+25], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+27], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+29], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+31], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+33], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+35], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+37], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+39], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+41], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+13], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+15], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+17], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+19], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+21], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+23], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+25], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+27], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+29], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+31], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+33], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+35], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+37], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+39], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+41], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+13], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+15], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+17], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+19], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+21], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+23], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+25], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+27], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+29], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+31], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+33], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+35], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+37], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+39], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+41], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+15], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+17], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+19], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+21], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+23], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+25], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+27], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+29], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+31], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+33], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+35], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+37], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+39], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+41], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+13], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+15], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+17], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+19], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+21], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+23], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+25], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+27], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+29], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+31], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+33], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+35], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+37], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+39], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+41], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+13], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+15], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+17], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+19], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+21], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+23], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+25], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+27], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+29], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+31], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+33], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+35], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+37], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+39], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+41], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+13], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+15], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+17], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+19], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+21], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+23], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+25], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+27], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+29], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+31], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+33], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+35], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+37], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+39], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+41], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_End_1: +s_getpc_b64 s[58:59] // addr of next instr +s_add_i32 s60, label_KernelEnd, 0x4 // target branch offset +s_add_u32 s58, s58, s60 // add target branch offset +s_addc_u32 s59, s59, 0 // add high and carry +s_setpc_b64 s[58:59] // branch to label_KernelEnd +label_GSU_5: +s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? +s_cbranch_scc0 label_ScaleAlphaVec_1AddrValid // branch if s[AddressScaleAlphaVec] != 0 +s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 +s_branch label_ScaleAlphaVec_1AddrValid_End +label_ScaleAlphaVec_1AddrValid: +s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] +label_ScaleAlphaVec_1AddrValid_End: + +s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE +s_add_u32 s8, s[sgprWorkGroup2], 0x1 +s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) +s_cmp_eq_u32 s8, 0x0 // bias stride = 0? +s_cselect_b32 s8, s[sgprSizeI], s8 +s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? +s_cbranch_scc0 label_Bias_1AddrValid // branch if s[AddressBias] != 0 +s_mov_b32 s[sgprSrdBias+2], 0 +s_branch label_Bias_1AddrValid_End +label_Bias_1AddrValid: +s_mov_b32 s[sgprSrdBias+2], s8 +label_Bias_1AddrValid_End: + +label_Load_Biasf32_0_1: +s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 +s_cbranch_scc1 label_Load_Biasbf16_0_1 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE +buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End_1 // Branch to load bias end +label_Load_Biasbf16_0_1: +s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 +s_cbranch_scc1 label_Load_Bias_End_1 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE +buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End_1 // Branch to load bias end +label_Load_Bias_End_1: +s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 +s_cbranch_scc0 label_GW_Beta_2 // Branch if Beta is not zero + +s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? +s_cselect_b32 s60, s60, 0 // set rMT0 +s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_PFO42GJLMDBXSWVP_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B0_E1_M_1, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_M_1 +label_NoBranch_PFO42GJLMDBXSWVP_0: +s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 +s_cselect_b32 s60, s60, 0 // set rMT1 +s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_XR5UBTMCR0HWLF5H_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B0_E1_N_1, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_N_1 +label_NoBranch_XR5UBTMCR0HWLF5H_0: +label_GW_B0_E0_2: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_0 // Branch if true +label_To_Activation_None_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Abs_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Gelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Relu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Tanh_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Silu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_ActivationSetPCAddrEnd_6: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_B0_E1_N_1: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_1 // Branch if true +label_To_Activation_None_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Abs_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Gelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Relu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Tanh_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Silu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_ActivationSetPCAddrEnd_5: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=14 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+25], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+26], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+27], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+68], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+69], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+70], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+71], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+76], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+77], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+78], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+79], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+80], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+81], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+82], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+83], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+88], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+89], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+90], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+91], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+96], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+97], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+98], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+99], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+104], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+105], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+106], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+107], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+108], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+109], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+110], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+111], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+116], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+117], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+118], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+119], acc61 // copy acc to vreg[55] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+25], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+26], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+27], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+32], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+33], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+34], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+35], acc63 // copy acc to vreg[63] +v_accvgpr_read_b32 v[vgprValuC+40], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+41], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+42], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+43], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+48], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+49], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+50], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+51], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+52], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+53], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+54], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+55], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+60], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+61], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+62], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+63], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+68], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+69], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+70], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+71], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+76], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+77], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+78], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+79], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+80], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+81], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+82], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+83], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+88], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+89], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+90], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+91], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+96], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+97], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+98], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+99], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+104], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+105], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+106], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+107], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+108], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+109], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+110], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+111], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+116], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+117], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+118], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+119], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+25], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+26], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+27], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+32], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+33], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+34], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+35], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+40], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+41], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+42], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+43], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+48], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+49], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+50], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+51], acc127 // copy acc to vreg[127] +v_accvgpr_read_b32 v[vgprValuC+52], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+53], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+54], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+55], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+60], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+61], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+62], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+63], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+68], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+69], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+70], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+71], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+76], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+77], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+78], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+79], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+80], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+81], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+82], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+83], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+88], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+89], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+90], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+91], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+96], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+97], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+98], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+99], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+104], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+105], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+106], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+107], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+108], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+109], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+110], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+111], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+116], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+117], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+118], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+119], acc173 // copy acc to vreg[167] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4); (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+25], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+26], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+27], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+32], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+33], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+34], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+35], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+40], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+41], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+42], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+43], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+48], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+49], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+50], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+51], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+52], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+53], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+54], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+55], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+60], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+61], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+62], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+63], acc191 // copy acc to vreg[191] +v_accvgpr_read_b32 v[vgprValuC+68], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+69], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+70], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+71], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+76], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+77], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+78], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+79], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+80], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+81], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+82], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+83], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+88], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+89], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+90], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+91], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+96], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+97], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+98], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+99], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+104], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+105], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+106], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+107], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+108], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+109], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+110], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+111], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+116], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+117], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+118], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+119], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0), (12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v73, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v73, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v73, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v73, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v73, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v73, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v73, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v73, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v73, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v73, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v73, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v73, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v73, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v73, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v73, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v73, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v73, v66, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+25], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+26], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+27], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+32], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+33], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+34], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+35], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+40], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+41], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+42], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+43], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+48], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+49], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+50], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+51], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+52], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+53], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+54], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+55], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+60], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+61], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+62], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+63], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+76], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+77], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+78], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+79], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_B0_E1_M_1: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_0_edge_1 // Branch if true +label_To_Activation_None_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Abs_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Clippedrelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Gelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Leakyrelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Relu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Sigmoid_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Tanh_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Geluscaling_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Silu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_ActivationSetPCAddrEnd_4: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+24], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+36], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+40], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+44], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+48], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+52], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+56], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+60], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+64], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+68], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+72], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+76], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+80], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+84], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+24], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+36], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+40], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+44], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+48], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+52], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+56], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+60], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+64], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+68], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+76], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+80], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+84], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+24], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+30], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+36], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+40], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+44], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+48], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+52], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+56], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+60], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+64], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+68], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+76], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+80], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+84], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+24], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+36], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+40], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+44], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+48], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+52], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+56], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+60], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+64], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+68], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+76], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+80], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+84], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+24], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+36], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+40], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+44], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+48], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+52], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+56], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+60], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+64], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+68], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+72], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+76], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+80], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+84], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+24], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+36], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+40], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+44], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+48], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+52], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+56], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+60], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+64], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+68], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+76], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+80], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+84], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+24], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+36], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+40], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+44], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+48], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+52], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+56], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+60], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+64], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+68], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+76], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+80], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+84], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+24], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+30], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+36], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+40], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+44], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+48], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+52], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+56], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+60], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+64], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+68], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+76], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+80], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+84], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+24], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+36], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+40], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+44], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+48], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+52], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+56], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+60], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+64], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+68], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+72], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+76], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+80], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+84], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+24], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+36], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+40], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+44], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+48], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+52], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+56], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+60], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+64], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+68], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+76], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+80], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+84], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+24], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+30], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+36], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+40], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+44], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+48], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+52], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+56], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+60], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+64], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+68], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+76], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+80], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+84], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+24], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+30], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+36], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+40], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+44], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+48], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+52], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+56], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+60], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+64], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+68], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+76], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+80], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+84], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+24], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+36], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+40], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+44], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+48], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+52], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+56], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+60], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+64], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+68], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+72], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+76], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+80], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+84], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+24], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+36], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+40], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+44], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+48], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+52], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+56], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+60], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+64], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+68], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+76], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+80], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+84], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+24], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+30], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+36], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+40], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+44], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+48], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+52], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+56], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+60], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+64], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+68], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+76], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+80], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+84], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+24], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+36], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+40], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+44], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+48], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+52], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+56], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+60], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+64], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+68], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+76], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+80], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+84], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_Beta_2: +s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? +s_cselect_b32 s60, s60, 0 // set rMT0 +s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_1L38YJQL3BUJ48XK_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B1_E1_M, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_M +label_NoBranch_1L38YJQL3BUJ48XK_0: +s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 +s_cselect_b32 s60, s60, 0 // set rMT1 +s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_XMVL70A9XU3BIJFQ_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B1_E1_N, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_N +label_NoBranch_XMVL70A9XU3BIJFQ_0: +label_GW_B1_E0: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_0 // Branch if true +label_To_Activation_None_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Abs_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Gelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Relu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Tanh_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Silu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_ActivationSetPCAddrEnd_3: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_add_lshl_u32 v14, v2, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+64], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+65], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+66], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+67], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+73], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+74], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+75], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+76], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+77], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+78], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+79], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+84], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+85], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+86], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+87], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+88], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+89], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+90], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+91], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+96], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+97], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+98], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+99], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+100], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+101], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+102], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+103], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+108], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+109], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+110], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+111], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+112], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+113], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+114], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+115], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+120], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+121], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+122], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+123], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+36], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+37], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+38], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+39], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+40], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+41], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+42], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+43], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+48], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+49], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+50], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+51], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+52], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+53], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+54], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+55], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+60], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+61], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+62], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+63], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+64], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+65], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+66], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+67], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+73], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+74], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+75], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+76], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+77], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+78], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+79], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+84], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+85], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+86], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+87], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+88], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+89], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+90], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+91], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+96], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+97], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+98], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+99], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+100], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+101], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+102], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+103], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+108], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+109], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+110], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+111], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+112], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+113], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+114], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+115], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+120], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+121], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+122], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+123], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+36], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+37], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+38], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+39], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+40], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+41], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+42], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+43], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+48], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+49], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+50], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+51], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+52], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+53], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+54], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+55], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+60], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+61], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+62], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+63], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+64], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+65], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+66], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+67], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+73], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+74], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+75], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+76], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+77], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+78], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+79], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+84], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+85], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+86], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+87], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+88], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+89], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+90], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+91], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+96], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+97], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+98], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+99], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+112], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+113], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+114], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+115], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+120], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+121], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+122], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+123], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+40], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+41], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+42], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+43], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+48], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+49], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+50], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+51], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+52], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+53], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+54], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+55], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+60], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+61], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+62], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+63], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+64], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+65], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+66], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+67], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+73], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+74], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+75], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+76], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+77], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+78], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+79], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+84], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+85], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+86], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+87], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+88], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+89], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+90], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+91], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+96], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+97], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+98], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+99], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+100], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+101], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+102], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+103], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+108], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+109], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+110], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+111], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+112], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+113], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+114], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+115], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+120], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+121], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+122], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+123], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_GW_B1_E1_N: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_1 // Branch if true +label_To_Activation_None_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Abs_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Gelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Relu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Tanh_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Silu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_ActivationSetPCAddrEnd_2: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=12 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+44], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+45], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+46], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+47], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+56], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+57], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+58], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+59], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+64], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+65], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+66], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+67], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+72], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+73], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+74], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+75], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+80], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+81], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+82], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+83], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+92], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+93], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+94], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+95], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+100], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+101], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+102], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+103], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+108], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+109], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+110], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+111], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+116], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+117], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+118], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+119], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+128], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+129], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+130], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+131], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+29], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+31], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+36], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+37], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+38], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+39], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+44], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+45], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+46], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+47], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+56], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+57], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+58], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+59], acc63 // copy acc to vreg[63] +v_accvgpr_read_b32 v[vgprValuC+64], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+65], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+66], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+67], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+72], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+73], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+74], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+75], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+80], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+81], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+82], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+83], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+92], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+93], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+94], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+95], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+100], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+101], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+102], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+103], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+108], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+109], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+110], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+111], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+116], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+117], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+118], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+119], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+128], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+129], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+130], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+131], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+29], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+31], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+36], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+37], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+38], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+39], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+44], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+45], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+46], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+47], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+64], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+65], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+66], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+67], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+72], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+73], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+74], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+75], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+80], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+81], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+82], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+83], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+92], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+93], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+94], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+95], acc127 // copy acc to vreg[127] +v_accvgpr_read_b32 v[vgprValuC+100], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+101], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+102], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+103], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+108], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+109], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+110], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+111], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+116], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+117], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+118], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+119], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+128], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+129], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+130], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+131], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+64], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+65], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+66], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+67], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+72], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+73], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+74], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+75], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+80], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+81], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+82], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+83], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+92], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+93], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+94], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+95], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+116], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+117], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+118], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+119], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+128], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+129], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+130], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+131], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+44], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+45], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+46], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+47], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+56], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+57], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+58], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+59], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+64], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+65], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+66], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+67], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+72], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+73], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+74], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+75], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+80], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+81], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+82], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+83], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+92], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+93], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+94], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+95], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+100], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+101], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+102], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+103], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+108], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+109], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+110], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+111], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+116], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+117], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+118], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+119], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+128], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+129], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+130], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+131], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v51, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v51, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v51, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v51, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v51, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v51, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v51, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v51, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v51, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v51, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v51, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v51, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v51, v48, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+29], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+31], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+36], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+37], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+38], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+39], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+44], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+45], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+46], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+47], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+56], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+57], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+58], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+59], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_GW_B1_E1_M: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_1_edge_1 // Branch if true +label_To_Activation_None_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Abs_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Clippedrelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Gelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Leakyrelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Relu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Sigmoid_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Tanh_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Geluscaling_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Silu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_ActivationSetPCAddrEnd_1: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+26], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+33], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+40], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+45], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+50], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+55], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+60], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+65], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+70], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+75], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+80], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+85], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+90], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+95], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+100], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+26], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+33], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+40], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+45], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+50], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+55], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+60], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+65], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+70], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+75], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+80], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+85], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+90], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+95], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+100], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+26], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+33], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+40], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+45], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+50], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+55], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+60], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+65], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+70], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+75], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+80], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+85], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+90], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+95], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+100], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+26], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+33], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+40], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+45], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+50], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+55], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+60], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+65], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+70], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+75], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+80], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+85], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+90], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+95], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+100], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+26], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+33], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+40], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+45], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+50], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+55], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+60], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+65], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+70], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+75], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+80], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+85], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+90], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+95], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+100], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+26], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+33], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+40], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+45], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+50], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+55], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+60], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+65], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+70], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+75], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+80], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+85], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+90], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+95], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+100], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+26], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+33], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+40], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+45], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+50], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+55], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+60], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+65], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+70], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+75], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+80], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+85], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+90], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+95], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+100], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+26], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+33], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+40], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+45], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+50], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+55], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+60], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+65], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+70], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+75], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+80], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+85], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+90], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+95], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+100], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+26], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+33], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+40], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+45], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+50], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+55], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+60], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+65], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+70], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+75], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+80], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+85], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+90], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+95], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+100], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+26], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+33], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+40], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+45], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+50], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+55], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+60], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+65], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+70], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+75], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+80], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+85], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+90], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+95], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+100], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+26], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+33], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+40], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+45], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+50], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+55], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+60], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+65], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+70], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+75], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+80], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+85], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+90], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+95], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+100], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+26], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+33], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+40], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+45], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+50], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+55], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+60], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+65], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+70], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+75], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+80], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+85], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+90], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+95], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+100], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+26], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+33], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+40], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+45], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+50], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+55], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+60], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+65], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+70], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+75], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+80], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+85], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+90], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+95], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+100], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+26], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+33], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+40], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+45], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+50], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+55], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+60], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+65], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+70], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+75], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+80], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+85], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+90], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+95], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+100], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+26], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+33], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+40], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+45], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+50], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+55], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+60], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+65], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+70], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+75], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+80], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+85], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+90], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+95], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+100], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+26], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+33], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+40], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+45], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+50], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+55], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+60], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+65], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+70], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+75], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+80], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+85], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+90], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+95], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+100], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_Activation_None_VW1: +s_setpc_b64 s[58:59] +label_Activation_Abs_VW1: +v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit +s_setpc_b64 s[58:59] +label_Activation_Clippedrelu_VW1: +v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) +v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha +s_setpc_b64 s[58:59] +label_Activation_Gelu_VW1: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +s_setpc_b64 s[58:59] +label_Activation_Leakyrelu_VW1: +v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha +v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? +v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 +s_setpc_b64 s[58:59] +label_Activation_Relu_VW1: +v_max_f32 v4, v4, 0 // x = max(0, x) +s_setpc_b64 s[58:59] +label_Activation_Sigmoid_VW1: +v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // 1 + exp(-x) +v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +s_setpc_b64 s[58:59] +label_Activation_Tanh_VW1: +v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha +v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // e^2x + 1 +v_rcp_f32 v4, v4 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) +s_setpc_b64 s[58:59] +label_Activation_Geluscaling_VW1: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +s_setpc_b64 s[58:59] +label_Activation_Silu_VW1: +v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) +s_setpc_b64 s[58:59] +label_GW_End_2: +label_KernelEnd: +s_endpgm // Kernel End +label_Activation_None_VW4: +s_setpc_b64 s[58:59] +label_Activation_Abs_VW4: +v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit +v_and_b32 v5, 0x7fffffff, v5 // Remove sign bit +v_and_b32 v6, 0x7fffffff, v6 // Remove sign bit +v_and_b32 v7, 0x7fffffff, v7 // Remove sign bit +s_setpc_b64 s[58:59] +label_Activation_Clippedrelu_VW4: +v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) +v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v5, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v5, s[sgpractivationBeta], v5 // min(x, beta) +v_cndmask_b32 v5, 0.0, v5, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v6, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v6, s[sgpractivationBeta], v6 // min(x, beta) +v_cndmask_b32 v6, 0.0, v6, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v7, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v7, s[sgpractivationBeta], v7 // min(x, beta) +v_cndmask_b32 v7, 0.0, v7, vcc // set x to 0 if <= alpha +s_setpc_b64 s[58:59] +label_Activation_Gelu_VW4: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v5 // k1 * x +v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) +v_mul_f32 v5, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v6 // k1 * x +v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) +v_mul_f32 v6, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v7 // k1 * x +v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) +v_mul_f32 v7, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +s_setpc_b64 s[58:59] +label_Activation_Leakyrelu_VW4: +v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha +v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? +v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v5 // tmp = x * alpha +v_cmp_ge_f32 vcc, v5, 0.0 // x >= 0 ? +v_cndmask_b32 v5, v8, v5, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v6 // tmp = x * alpha +v_cmp_ge_f32 vcc, v6, 0.0 // x >= 0 ? +v_cndmask_b32 v6, v8, v6, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v7 // tmp = x * alpha +v_cmp_ge_f32 vcc, v7, 0.0 // x >= 0 ? +v_cndmask_b32 v7, v8, v7, vcc // set x to tmp if < 0 +s_setpc_b64 s[58:59] +label_Activation_Relu_VW4: +v_max_f32 v4, v4, 0 // x = max(0, x) +v_max_f32 v5, v5, 0 // x = max(0, x) +v_max_f32 v6, v6, 0 // x = max(0, x) +v_max_f32 v7, v7, 0 // x = max(0, x) +s_setpc_b64 s[58:59] +label_Activation_Sigmoid_VW4: +v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // 1 + exp(-x) +v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v5, 0xbfb8aa3b, v5 // (fused -1.442695) +v_exp_f32 v5, v5 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v5, 1.0, v5 // 1 + exp(-x) +v_rcp_f32 v5, v5 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v6, 0xbfb8aa3b, v6 // (fused -1.442695) +v_exp_f32 v6, v6 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v6, 1.0, v6 // 1 + exp(-x) +v_rcp_f32 v6, v6 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v7, 0xbfb8aa3b, v7 // (fused -1.442695) +v_exp_f32 v7, v7 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v7, 1.0, v7 // 1 + exp(-x) +v_rcp_f32 v7, v7 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +s_setpc_b64 s[58:59] +label_Activation_Tanh_VW4: +v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha +v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // e^2x + 1 +v_rcp_f32 v4, v4 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) +v_mul_f32 v5, s[sgpractivationAlpha], v5 // x * alpha +v_mul_f32 v5, 0x4038aa3b, v5 // (fused 2) +v_exp_f32 v5, v5 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v5, 1.0, v5 // e^2x + 1 +v_rcp_f32 v5, v5 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v5, -2.0, v5, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v5, s[sgpractivationBeta], v5 // beta * tanh(x) +v_mul_f32 v6, s[sgpractivationAlpha], v6 // x * alpha +v_mul_f32 v6, 0x4038aa3b, v6 // (fused 2) +v_exp_f32 v6, v6 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v6, 1.0, v6 // e^2x + 1 +v_rcp_f32 v6, v6 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v6, -2.0, v6, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v6, s[sgpractivationBeta], v6 // beta * tanh(x) +v_mul_f32 v7, s[sgpractivationAlpha], v7 // x * alpha +v_mul_f32 v7, 0x4038aa3b, v7 // (fused 2) +v_exp_f32 v7, v7 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v7, 1.0, v7 // e^2x + 1 +v_rcp_f32 v7, v7 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v7, -2.0, v7, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v7, s[sgpractivationBeta], v7 // beta * tanh(x) +s_setpc_b64 s[58:59] +label_Activation_Geluscaling_VW4: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v5 // k1 * x +v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v5, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v6 // k1 * x +v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v6, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v7 // k1 * x +v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v7, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +s_setpc_b64 s[58:59] +label_Activation_Silu_VW4: +v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v5 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v5, v5, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v6 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v6, v6, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v7 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v7, v7, v8 // x / (1 + exp(-x)) +s_setpc_b64 s[58:59] +s_endpgm +label_ASM_End: /// The end of the kernel diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s index 586ab4fbc1..8ff3e883fa 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s @@ -56,7 +56,27 @@ custom.config: TransposeB: 0 UseBeta: True Batched: True - MatrixInstruction: [16, 16,16, 1, 1, 2,1, 4,1 ] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] AssertFree0ElementMultiple: 16 AssertFree1ElementMultiple: 1 AssertSummationElementMultiple: 128 From 5576c7e69b87e307e44658b623887c6772e991ec Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 28 Feb 2025 17:08:58 +0000 Subject: [PATCH 085/105] draft: TCL is building --- tensilelite/Tensile/BenchmarkProblems.py | 4 +- tensilelite/Tensile/Common/ValidParameters.py | 3 ++ tensilelite/Tensile/CustomKernels.py | 4 +- tensilelite/Tensile/KernelWriter.py | 6 +-- tensilelite/Tensile/KernelWriterAssembly.py | 26 ++++----- tensilelite/Tensile/LibraryLogic.py | 54 +++++++++---------- tensilelite/Tensile/SolutionLibrary.py | 36 ++++++------- .../Tensile/SolutionSelectionLibrary.py | 4 +- .../{Common => SolutionStructs}/Naming.py | 7 +-- .../Tensile/SolutionStructs/Solution.py | 8 +-- .../Tensile/TensileCreateLibrary/Run.py | 4 +- 11 files changed, 80 insertions(+), 76 deletions(-) rename tensilelite/Tensile/{Common => SolutionStructs}/Naming.py (98%) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 4c4a1bfeb7..45114f6c50 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -37,6 +37,8 @@ from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler from Tensile.SolutionStructs import Solution, ProblemSizes, matrixInstructionToMIParameters +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ + getKeyNoInternalArgs from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType @@ -50,8 +52,6 @@ from .Common import globalParameters, HR, print1, print2, IsaInfo, defaultSolution, \ printExit, printWarning, ensurePath, startTime, tqdm, state, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, DepthUConfig -from Tensile.Common.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ - getKeyNoInternalArgs def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index cd43cc10ea..fecbf6b49f 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -782,6 +782,9 @@ "ConvertAfterDS": [False, True], # Force disable shadow init to release more sgpr in preloop "ForceDisableShadowInit": [False, True], +} + +newMIValidParameters = { "EnableF32XdlMathOp": [False, True], 'EnableMatrixInstruction': [False, True], 'ISA': -1, diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index bd7a9ba18e..2022102792 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -23,7 +23,7 @@ ################################################################################ from . import CUSTOM_KERNEL_PATH -from .Common.ValidParameters import checkParametersAreValid, validParameters +from .Common.ValidParameters import checkParametersAreValid, validParameters, newMIValidParameters import yaml @@ -95,7 +95,7 @@ def getCustomKernelConfig( kernelIsp[key] = internalSupportParams[key] for k, v in kernelConfig.items(): if k != "ProblemType": - checkParametersAreValid((k, [v]), validParameters) + checkParametersAreValid((k, [v]), validParameters + newMIValidParameters) kernelConfig["KernelLanguage"] = "Assembly" kernelConfig["CustomKernelName"] = kernelName diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 84566ccd05..aee4368cec 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -36,8 +36,8 @@ from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ - INDEX_CHARS, IsaVersion -from Tensile.Common.Naming import getKernelName + INDEX_CHARS, IsaVersion +from Tensile.SolutionStructs.Naming import getKernelName from Tensile.Toolchain.Component import Assembler import abc @@ -365,7 +365,7 @@ def __init__( kernelMinNaming, kernelSerialNaming, assembler: Assembler, - debugConfig: DebugConfig, + debugConfig: DebugConfig, ): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 0d3bebd814..1724c3ea7e 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -49,7 +49,7 @@ from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig from Tensile.Common import print1, print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection -from Tensile.Common.Naming import getKernelFileBase +from Tensile.SolutionStructs.Naming import getKernelFileBase from Tensile.Toolchain.Component import Assembler from math import ceil, log, floor @@ -88,7 +88,7 @@ def __init__( kernelMinNaming, kernelSerialNaming, assembler: Assembler, - debugConfig: DebugConfig, + debugConfig: DebugConfig, ): super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, debugConfig) @@ -115,8 +115,8 @@ def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): return code - def getSourceFileString(self, - kernel, + def getSourceFileString(self, + kernel, useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" # Skip if .o files will have already been built for this file @@ -4401,7 +4401,7 @@ def tailLoopAllocDTVVgpr(self, kernel, tensorParametersA, tensorParametersB): numVgprValuPackA *= 2 else: numVgprValuPackA = self.states.a.numVgprValuPerBlock * kernel["InnerUnroll"] * self.states.numVgprBufferPackA * (int(4/tensorParametersA["bpeDS"]) - 1) - + vgprBaseA = self.vgprPool.checkOutAligned(numValuA + numVgprValuPackA, 2) imodA.add(RegSet("v", "vgprValuA_X0_I0_BASE", vgprBaseA)) if numVgprValuPackA > 0: @@ -4477,7 +4477,7 @@ def tailLoopAllocDTVVgpr(self, kernel, tensorParametersA, tensorParametersB): ############################################################################## # Using wider load instructions to improve the GR efficiency in tail loop. # If loading size is smaller than a dword(32bit), it will return 0 instead. - # Need to call buffer_load_d16 to load the data which is out of boundary. + # Need to call buffer_load_d16 to load the data which is out of boundary. ############################################################################## def tailLoopGlobalRead(self, kernel, tPA, tPB, doA, doB): imod = Module("tailLoopGlobalRead") @@ -4717,7 +4717,7 @@ def func(idx, bevavior, jumpLabel, tileSgpr, kSgpr): if doA and kernel["DirectToLds%s"%tPA["tensorChar"]]: imod.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumBytes"]), \ comment="Restore LDS clamp at %u bytes HERE"%(kernel["LdsNumBytes"]))) - + imod.add(SCmpEQU32(src0=sgpr(tmpSgprKB), src1=0, \ comment="Valid loading size per thread is multiples of 4 bytes")) @@ -5526,7 +5526,7 @@ def endSummation(self, kernel, tPA, tPB, noSkipLoad = True, label = None, isOptN # Write bias A, B data to LDS if kernel["ProblemType"]["Gradient"] and kernel["ProblemType"]["UseBias"] and (kernel["ProblemType"]["BiasSrc"] == "A" or kernel["ProblemType"]["BiasSrc"] == "B"): - + tP = tPA if kernel["ProblemType"]["BiasSrc"] == "A" else tPB module.add(self.exclasses.biasSumUnroll.storeSumLDS(self, kernel, tP)) @@ -7449,17 +7449,17 @@ def globalReadGuardKBody(tP, optParams = None): if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: if tP["is_sparse"]: globalReadGuardKBody(tP["tpsMetadata"]) - + if self.db["ConservativeWaitCnt"] & 0x1: module.add(SBarrier(comment="debug")) module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) module.add(SBarrier(comment="debug")) - + # TODO - can remove one of these m0 restores if A and B both TLU if kernel["DirectToLds%s"%tP["tensorChar"]]: module.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumBytes"]), \ comment="Restore LDS clamp at %u bytes HERE"%(kernel["LdsNumBytes"]))) - + if not kernel["BufferLoad"]: self.vgprPool.checkIn(maxAddrVgpr) self.vgprPool.checkIn(bpeVgpr) @@ -7650,7 +7650,7 @@ def globalReadDo(self, kernel, mode, tP, unrollLoopIdx=-1, g2lBufIdx=0, \ tc = tP["tensorChar"] problemType = self.states.kernel["ProblemType"] imod = StructuredModule("globalReadDo%s_%u"%(tc,mode)) - if not self.do["GlobalRead%s"%tP["tensorChar"]]: + if not self.do["GlobalRead%s"%tP["tensorChar"]]: return imod # sizeK % LOCAL_DEPTHU @@ -7810,7 +7810,7 @@ def globalReadBody(tP): else: g2lIdxM = i * max(loadWidth * tP["bpeRatio"], 1) destVgpr = destVgprPrefix + "+%u"%((g2lIdx+eccOffset+tP["shiftGR"]) if not tP["isM"] else g2lIdxM) - self.vgprs.globalReadRegisters[tc].append(g2lIdx+eccOffset+tP["shiftGR"] if not tP["isM"] else g2lIdxM) + self.vgprs.globalReadRegisters[tc].append(g2lIdx+eccOffset+tP["shiftGR"] if not tP["isM"] else g2lIdxM) if tP["isM"]: assert(graIdx <= self.states.m.numVgprG2LAllocated) diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index 8b3b4f0e90..fa6fe6298c 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -27,7 +27,7 @@ from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig -from Tensile.Common.Naming import getMinNaming, getNameMin, getNameFull +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin, getNameFull from . import LibraryIO from . import SolutionSelectionLibrary @@ -1432,13 +1432,13 @@ def indicesToSerial(self, solutionIdx, indices ): def generateLogic( - config, - benchmarkDataPath, - libraryLogicPath, - cxxCompiler: str, - splitGSU: bool, - printSolutionRejectionReason: bool, - printIndexAssignmentInfo: bool, + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler: str, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo] ): @@ -1485,11 +1485,11 @@ def generateLogic( if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) (problemSizes, solutions) = LibraryIO.parseSolutionsFile( - solutionsFileName, - cxxCompiler, - splitGSU, - printSolutionRejectionReason, - printIndexAssignmentInfo, + solutionsFileName, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, depthUConfig, isaInfoMap ) @@ -1566,25 +1566,25 @@ def read_max_freq(): ################################################################################ ################################################################################ def main( - config, - cxxCompiler: str, - outputPath: Path, - splitGSU: bool, - printSolutionRejectionReason: bool, - printIndexAssignmentInfo: bool, + config, + cxxCompiler: str, + outputPath: Path, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo] ): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR generateLogic( - config, - benchmarkDataPath, - libraryLogicPath, - cxxCompiler, - splitGSU, - printSolutionRejectionReason, - printIndexAssignmentInfo, - depthUConfig, + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, isaInfoMap ) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 96fcbe2ab2..258baae39a 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -30,7 +30,7 @@ from . import Contractions from .SolutionStructs import Solution as OriginalSolution from .Common import state, IsaInfo, gfxToIsa, DepthUConfig -from Tensile.Common.Naming import getMinNaming, getNameMin +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin class SingleSolutionLibrary: Tag = "Single" @@ -482,12 +482,12 @@ def selection(d, problemType, solutions, library, placeholderName): problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) allSolutions = [solutionClass.FromSolutionStruct( - s, - splitGSU, - printSolutionRejectionReason, + s, + splitGSU, + printSolutionRejectionReason, printIndexAssignmentInfo, - depthUConfig, - assembler, + depthUConfig, + assembler, isaInfoMap ) for s in origSolutions] cls.FixSolutionIndices(allSolutions) @@ -511,23 +511,23 @@ def selection(d, problemType, solutions, library, placeholderName): @classmethod def BenchmarkingLibrary( - cls, - solutions, - assembler, - splitGSU: bool, - printSolutionRejectionReason: bool, - printIndexAssignmentInfo: bool, + cls, + solutions, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, depthUConfig: DepthUConfig, isaInfoMap ): solutionObjs = list([Contractions.Solution.FromOriginalState( - s._state, - splitGSU, - printSolutionRejectionReason, - printIndexAssignmentInfo, + s._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, depthUConfig, - assembler, - isaInfoMap) + assembler, + isaInfoMap) for s in solutions]) cls.FixSolutionIndices(solutionObjs) diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py index 49a8115e76..df7dd17bdd 100644 --- a/tensilelite/Tensile/SolutionSelectionLibrary.py +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ # ################################################################################ -from Tensile.Common.Naming import getNameMin +from Tensile.SolutionStructs.Naming import getNameMin import csv diff --git a/tensilelite/Tensile/Common/Naming.py b/tensilelite/Tensile/SolutionStructs/Naming.py similarity index 98% rename from tensilelite/Tensile/Common/Naming.py rename to tensilelite/Tensile/SolutionStructs/Naming.py index 1da89b4751..fe7de4cbc9 100644 --- a/tensilelite/Tensile/Common/Naming.py +++ b/tensilelite/Tensile/SolutionStructs/Naming.py @@ -23,14 +23,15 @@ ################################################################################ from copy import deepcopy from functools import lru_cache -from .Constants import MAX_FILENAME_LENGTH -from .ValidParameters import validParameters +from typing import List +from Tensile.Common.Constants import MAX_FILENAME_LENGTH +from Tensile.Common.ValidParameters import validParameters from Tensile.ProblemType import ProblemType ######################################## # create a dictionary with booleans on whether to include parameter in name -def getMinNaming(objs): +def getMinNaming(objs: list): nonCKObjs = [obj for obj in objs if not ("CustomKernelName" in obj and obj["CustomKernelName"])] # early return if len(nonCKObjs) == 0: diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 63f996af1e..90084dd72f 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -43,7 +43,7 @@ print1, print2, printExit, printWarning, \ roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ DepthUConfig -from Tensile.Common.Naming import getNameFull +from Tensile.SolutionStructs.Naming import getNameFull from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler @@ -208,9 +208,9 @@ def __init__( else: self["InternalSupportParams"] = defaultInternalSupportParams - # # assign parameters with defaults - # for key in defaultSolution: - # assignParameterWithDefault(self._state, key, config, defaultSolution) + # Assign solution state from config, filling missing from the defaultSolution + for key in defaultSolution: + assignParameterWithDefault(self._state, key, config, defaultSolution) if 'ISA' not in self._state: if 'ISA' in config: diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 8ecff0e3c0..72e3ba87e0 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -55,7 +55,7 @@ tqdm, verbosity, ) -from Tensile.Common.Naming import getKernelFileBase, getKeyNoInternalArgs, getMinNaming, getSerialNaming +from Tensile.SolutionStructs.Naming import getKernelFileBase, getKeyNoInternalArgs, getMinNaming, getSerialNaming from Tensile.CustomYamlLoader import load_logic_gfx_arch from Tensile.KernelWriterAssembly import KernelWriterAssembly @@ -366,7 +366,7 @@ def assemble(ret): asmToolchain.linker, asmToolchain.bundler, globalParameters["ROCmLdPath"], - asmKernels, + asmKernels, destLibPath, assemblyTmpPath, compress, From 7670dd1741799e3436b4083d7fa43639836d1a91 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 28 Feb 2025 21:33:32 +0000 Subject: [PATCH 086/105] draft: printing and debugging, mostly --- tensilelite/Tensile/BenchmarkProblems.py | 10 ++++++---- tensilelite/Tensile/Common/Architectures.py | 19 +++++++++++++++++++ tensilelite/Tensile/Common/Capabilities.py | 2 +- .../Tensile/Components/ComputeStoreVgprs.py | 5 +++-- tensilelite/Tensile/ProblemType.py | 5 +++-- .../Tensile/SolutionStructs/Solution.py | 1 + .../Tensile/SolutionStructs/Utilities.py | 2 +- .../Tensile/TensileCreateLibrary/Run.py | 1 + tensilelite/Tensile/Toolchain/Assembly.py | 4 +++- tensilelite/Tensile/Toolchain/Source.py | 1 + 10 files changed, 39 insertions(+), 11 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 45114f6c50..5d6f0b7a20 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -49,8 +49,8 @@ from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain -from .Common import globalParameters, HR, print1, print2, IsaInfo, defaultSolution, \ - printExit, printWarning, ensurePath, startTime, tqdm, state, \ +from .Common import globalParameters, HR, print1, print2, IsaInfo, IsaVersion, \ + printExit, printWarning, ensurePath, startTime, tqdm, state, gfxToVariants, \ BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, DepthUConfig @@ -199,7 +199,7 @@ def writeBenchmarkFiles( debugConfig: DebugConfig, depthUConfig: DepthUConfig, deviceId: int, - isaInfoMap: Dict[str, IsaInfo] + isaInfoMap: Dict[IsaVersion, IsaInfo] ): """Write all the files needed for a given benchmarking step""" @@ -236,6 +236,8 @@ def writeBenchmarkFiles( debugConfig, ) + cmdLineArchs = [var for isa in isaInfoMap.keys() for var in gfxToVariants(isaToGfx(isa))] + # cmdLineArchs = [variant isaToGfx(isa) for isa in isaInfoMap.keys() for gfxToVariants()] # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ @@ -247,7 +249,7 @@ def writeBenchmarkFiles( kernelHelperObjs, kernelWriterAssembly, debugConfig.splitGSU, - [isaToGfx(isa) for isa in isaInfoMap.keys()], + cmdLineArchs, kernelSerialNaming, kernelMinNaming, errorTolerant=True, diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index 7559142554..e13ed074c3 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -56,6 +56,13 @@ "gfx1201": "gfx1201", } +gfxVariantMap = { + "gfx906": ["gfx906:xnack+", "gfx906:xnack-"], + "gfx908": ["gfx908:xnack+", "gfx908:xnack-"], + "gfx90a": ["gfx90a:xnack+", "gfx90a:xnack-"], + "gfx942": ["gfx942:xnack+", "gfx942:xnack-"], +} + SUPPORTED_ISA = [ IsaVersion(8, 0, 3), IsaVersion(9, 0, 0), @@ -137,6 +144,18 @@ def gfxToSwCodename(gfxName: str) -> Optional[str]: return None +def gfxToVariants(gfx: str) -> List[str]: + """Retrieves the list of variants for a given gfx architecture name. + + Args: + gfx: The name of the GPU architecture (e.g., 'gfx906'). + + Returns: + List of variants for the GPU architecture. + """ + return gfxVariantMap.get(gfx, [gfx]) + + def cliArchsToIsa(cliArchs: str) -> List[IsaVersion]: """Maps the requested gfx architectures to ISA numbers. diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index 1d2f6d8f39..00dcde5458 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -245,4 +245,4 @@ def makeIsaInfoMap(targetIsas: List[IsaVersion], cxxCompiler: str) -> Dict[IsaVe regCaps = initRegisterCaps(v, archCaps) asmBugs = initAsmBugs(asmCaps) isaInfoMap[v] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) - return isaInfoMap \ No newline at end of file + return isaInfoMap diff --git a/tensilelite/Tensile/Components/ComputeStoreVgprs.py b/tensilelite/Tensile/Components/ComputeStoreVgprs.py index 5209c34660..bd2651715c 100644 --- a/tensilelite/Tensile/Components/ComputeStoreVgprs.py +++ b/tensilelite/Tensile/Components/ComputeStoreVgprs.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ vectorStaticRemainder, RegisterPoolResource, vgpr, sgpr, log2, \ vectorStaticDivideAndRemainder from ..Component import ComputeStoreVgprs -from ..Common import DataDirection +from ..Common import DataDirection, printExit, printWarning class ComputeStoreVgprsVALU(ComputeStoreVgprs): kernel = {"EnableMatrixInstruction": False, @@ -186,6 +186,7 @@ def __call__(self, writer, kernel): strideD1 = "StrideD%s" % (writer.states.indexChars[packedC1[0]]) module.add(VMulLOU32(dst=vgpr(writer.vgprs.cinRowPtr), src0=vgpr(lsuTid1), src1=sgpr(strideC1), comment=" offset 1")) module.add(VMulLOU32(dst=vgpr(writer.vgprs.coutRowPtrD), src0=vgpr(lsuTid1), src1=sgpr(strideD1), comment=" offset 1")) + printWarning(f" --ERR-- problem: {kernel['ProblemType']}, GlobalSplitU: {kernel['GlobalSplitU']}, UseE: {kernel['ProblemType']['UseE']}") if kernel["ProblemType"]["UseE"] and (kernel["GlobalSplitU"] == 1): module.add(VMovB32(dst=vgpr(writer.vgprs.coutRowPtrE), src=vgpr(lsuTid1), comment=" save offset 1 for E")) if writer.vgprs.coutRowPtrBias != -1: diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py index 27c3e9e9ea..63a23fcd1c 100644 --- a/tensilelite/Tensile/ProblemType.py +++ b/tensilelite/Tensile/ProblemType.py @@ -29,7 +29,7 @@ from Tensile.TensileInstructions.DataType import DataType from Tensile.Common.Constants import INDEX_CHARS -from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2 +from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print ################################################################################ @@ -410,6 +410,7 @@ def __init__(self, config, printIndexAssignmentInfo: bool): if "Gradient" in config: if config["Gradient"]: + print1(f" --ERR-- Use bias and activation: {self['UseBias']} and {self['ActivationType']}") if (not self["UseBias"]) and self["ActivationType"] == 'none': printWarning("Gradient is disabled cause bias and activation are both disabled.") self["Gradient"] = False @@ -768,4 +769,4 @@ def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: biasDataTypeList = list(set(bList)) biasDataTypeList.sort() # Make name unique - return biasDataTypeList \ No newline at end of file + return biasDataTypeList diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 90084dd72f..cced3c5a35 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -514,6 +514,7 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, reject(state, printRejectionReason, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") # macro tile sizes + printWarning(f" ---ERR--- seting macro tile: {state['SubGroup0']} {state['ThreadTile0']} {state['SubGroup1']} {state['ThreadTile1']}") if "SubGroup0" in state and "ThreadTile0" in state: state["MacroTile0"] = state["SubGroup0"]*state["ThreadTile0"] if "SubGroup1" in state and "ThreadTile1" in state: diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index 4f0be9e675..053b8e85ac 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -159,5 +159,5 @@ def matrixInstructionToMIParameters( result['MIInputPerThreadMetadata'] = result['MIInputPerThread'] if not isSparse else result['MIInputPerThread'] // 8 result['Sparse'] = isSparse - print1(f">> MI Parameters: {pprint.pformat(result)}") + print2(f">> MI Parameters: {pprint.pformat(result)}") return result diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 72e3ba87e0..1b776112fa 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -285,6 +285,7 @@ def assemble(ret): assemblyTmpPath, compress, ) + print1(f" ---DBG--- Running buildSourceCodeObjectFiles cmdLineArchs: {cmdlineArchs}") buildSourceCodeObjectFiles( srcToolchain.compiler, srcToolchain.bundler, diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 831c28b7c6..a714d720e5 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -30,7 +30,7 @@ from pathlib import Path from typing import List, Union, NamedTuple -from ..Common import print2, isaToGfx +from ..Common import print2, isaToGfx, print1 from ..SolutionStructs import Solution from .Component import Assembler, Linker, Bundler @@ -115,6 +115,8 @@ def buildAssemblyCodeObjectFiles( coName = kernel.get("codeObjectFile", None) if coName: coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (kernel["BaseName"] + extObj))) + + print1(f" ---DBG--- CodeObjectFileMap: {coFileMap}, compress: {compress}") for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) linker(objFiles, str(coFileRaw)) diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 3cd54c5b35..7a50e087cf 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -58,6 +58,7 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa """ coPath = None buildPath = Path(buildPath) + print1(f" ---DBG--- target: {target}, base: {base}, buildPath: {buildPath}, arch: {arch}") if "TensileLibrary" in base and "fallback" in base: coPath = buildPath / "{0}_{1}.hsaco.raw".format(base, arch) elif "TensileLibrary" in base: From f7e41d469749f32dcf14973a61a25b6d98e9c15f Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Fri, 28 Feb 2025 23:18:50 +0000 Subject: [PATCH 087/105] draft: more printing and debugging tox tests --- tensilelite/Tensile/BenchmarkProblems.py | 1 + tensilelite/Tensile/ProblemType.py | 2 +- tensilelite/Tensile/SolutionStructs/Solution.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 5d6f0b7a20..c0e484300f 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -92,6 +92,7 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse assembler, isaInfoMap ) + print1(f" ---INF--- Solution: {solutionObject._state}") if solutionObject["Valid"]: solutionSet.add(solutionObject) elif debugConfig.printSolutionRejectionReason: diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py index 63a23fcd1c..8a2a66631c 100644 --- a/tensilelite/Tensile/ProblemType.py +++ b/tensilelite/Tensile/ProblemType.py @@ -29,7 +29,7 @@ from Tensile.TensileInstructions.DataType import DataType from Tensile.Common.Constants import INDEX_CHARS -from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print +from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print1 ################################################################################ diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index cced3c5a35..30ac0996c0 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -523,6 +523,8 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, if state["MacroTile0"] != state["MacroTile"][0] \ or state["MacroTile1"] != state["MacroTile"][1]: reject(state, printRejectionReason, "MacroTile mismatch") + printWarning(f" ---ERR--- seting macro tile: {state['MacroTile0']} {state['MacroTile1']}") + printWarning(f" ---ERR--- global split U: {state['GlobalSplitU']}") # tail loop optimization state["tailLoopOptA"] = True From 43b094bc57f6887a685a7a8ab31fafd302e5498e Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:32:26 +0000 Subject: [PATCH 088/105] fix: stubborn client bug --- tensilelite/Tensile/BenchmarkProblems.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index c0e484300f..19436f3fa7 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -59,7 +59,7 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") - # solutions = [] + solutions = [] solutionSet = set() for perm in forkPermutations: # Expect only a single ISA in the map for the Tensile context @@ -94,11 +94,13 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse ) print1(f" ---INF--- Solution: {solutionObject._state}") if solutionObject["Valid"]: - solutionSet.add(solutionObject) + if solutionObject not in solutionSet: + solutionSet.add(solutionObject) + solutions.append(solutionObject) elif debugConfig.printSolutionRejectionReason: print1("rejecting solution " + str(solution)) - return list(solutionSet) + return solutions def _getCustomKernelSolutionObj( From 8e4707c9d4d8f54481f6b6b33975ce67c0a8dc9d Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:34:57 +0000 Subject: [PATCH 089/105] refactor: problem type into solution struct --- tensilelite/Tensile/BenchmarkProblems.py | 4 +- tensilelite/Tensile/BenchmarkStructs.py | 2 +- tensilelite/Tensile/ClientWriter.py | 6 +- tensilelite/Tensile/Common/Utilities.py | 11 +- tensilelite/Tensile/LibraryIO.py | 2 +- tensilelite/Tensile/SolutionStructs/Naming.py | 3 +- .../Tensile/SolutionStructs/Problem.py | 748 ++++++++++++++++++ .../Tensile/SolutionStructs/Solution.py | 2 +- .../Tensile/SolutionStructs/__init__.py | 3 +- tensilelite/Tensile/TensileClientConfig.py | 3 +- .../Tensile/TensileLogic/ParseArguments.py | 2 +- tensilelite/Tensile/TensileLogic/Run.py | 143 +++- .../TensileLogic/ValidMatrixInstruction.py | 128 +-- .../Tensile/TensileLogic/ValidWorkGroup.py | 14 +- .../Tensile/TensileLogicCUSTOM/Utilities.py | 34 - .../ParseArguments.py | 1 - .../Run.py | 138 +--- .../Utilities.py | 0 .../ValidMatrixInstruction.py | 7 +- .../ValidWorkGroup.py | 0 .../__init__.py | 0 tensilelite/Tensile/bin/TensileLogic | 6 +- 22 files changed, 931 insertions(+), 326 deletions(-) delete mode 100644 tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py rename tensilelite/Tensile/{TensileLogicCUSTOM => TensileLogicOLD}/ParseArguments.py (94%) rename tensilelite/Tensile/{TensileLogicCUSTOM => TensileLogicOLD}/Run.py (51%) rename tensilelite/Tensile/{TensileLogic => TensileLogicOLD}/Utilities.py (100%) rename tensilelite/Tensile/{TensileLogicCUSTOM => TensileLogicOLD}/ValidMatrixInstruction.py (98%) rename tensilelite/Tensile/{TensileLogicCUSTOM => TensileLogicOLD}/ValidWorkGroup.py (100%) rename tensilelite/Tensile/{TensileLogicCUSTOM => TensileLogicOLD}/__init__.py (100%) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 19436f3fa7..35eb2964ec 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -34,9 +34,9 @@ from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters -from Tensile.ProblemType import ProblemType from Tensile.Toolchain.Component import Assembler -from Tensile.SolutionStructs import Solution, ProblemSizes, matrixInstructionToMIParameters +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizes +from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ getKeyNoInternalArgs diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index 849dc5c205..8dd27925e5 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -29,7 +29,7 @@ from Tensile.Common import print1, print2, hasParam, printExit, \ defaultBenchmarkCommonParameters, validParameters, globalParameters, \ defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes -from Tensile.ProblemType import ProblemType +from Tensile.SolutionStructs.Problem import ProblemType from .CustomKernels import getAllCustomKernelNames from .SolutionStructs import ProblemSizes, ActivationArgs, BiasTypeArgs, \ diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index a940f7b2cc..8126b7e043 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -32,16 +32,18 @@ from glob import glob from typing import List +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizesMock, ProblemSizesMockDummy +from Tensile.SolutionStructs import ActivationArgs, BiasTypeArgs, FactorDimArgs +from Tensile.Toolchain.Component import Assembler + from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, IsaInfo, \ LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR, detectGlobalCurrentISA, DepthUConfig -from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex from .Contractions import ProblemType as ContractionsProblemType -from Tensile.Toolchain.Component import Assembler class DataInitName(Enum): Zero = 0 diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 1746b0f5f0..f5679d5162 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -28,10 +28,11 @@ import sys import time +from inspect import currentframe, getframeinfo from copy import deepcopy from enum import Enum +from pathlib import Path -from .Types import IsaInfo from Tensile import __version__ verbosity = 1 @@ -109,6 +110,14 @@ def roundUp(f): return (int)(math.ceil(f)) +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" + + ################################################################################ # Is query version compatible with current version # a yaml file is compatible with tensile if diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index 2f45dd1c1e..256d530b4d 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -24,7 +24,7 @@ from .CustomKernels import getCustomKernelConfig from .SolutionStructs import Solution, ProblemSizes -from Tensile.ProblemType import ProblemType +from Tensile.SolutionStructs.Problem import ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream from .Common import gfxToIsa, printExit, printWarning, print2, \ diff --git a/tensilelite/Tensile/SolutionStructs/Naming.py b/tensilelite/Tensile/SolutionStructs/Naming.py index fe7de4cbc9..41c15c0828 100644 --- a/tensilelite/Tensile/SolutionStructs/Naming.py +++ b/tensilelite/Tensile/SolutionStructs/Naming.py @@ -27,7 +27,8 @@ from Tensile.Common.Constants import MAX_FILENAME_LENGTH from Tensile.Common.ValidParameters import validParameters -from Tensile.ProblemType import ProblemType + +from .Problem import ProblemType ######################################## # create a dictionary with booleans on whether to include parameter in name diff --git a/tensilelite/Tensile/SolutionStructs/Problem.py b/tensilelite/Tensile/SolutionStructs/Problem.py index 940afe2167..8f5b34cfa8 100644 --- a/tensilelite/Tensile/SolutionStructs/Problem.py +++ b/tensilelite/Tensile/SolutionStructs/Problem.py @@ -23,8 +23,17 @@ ################################################################################ from collections import OrderedDict +from collections.abc import Mapping + +from typing import List from Tensile.TensileInstructions.Base import fastdeepcopy as deepcopy +from Tensile.Activation import ActivationType +from Tensile.TensileInstructions.DataType import DataType +from Tensile.Common.Constants import INDEX_CHARS +from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print1, printExit + + class ProblemSizeRange: @@ -365,4 +374,743 @@ def __str__(self): s += " %s" % sizeRange return s +################################################################################ +# ProblemType +# name of solution should begin with name of problemType, and arguments can be listed out explicitly + +################################################################################ +# Default Problem Type +################################################################################ +_defaultProblemType = { + # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users + # =TensorContraction requires specifying + "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. + # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) + "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter + "UseE": False, # =True use output E to output gemm results before activation + "Gradient": False, # =True set globalWriteElements to gradient mode + "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction + "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. + "UseScaleAB": "", # Support "", "Scalar", and "Vector" + "UseScaleCD": False, # =True use scaleC, scaleD + "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction + "HighPrecisionAccumulate": False, # f32 += f16*f16 + "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. + "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case + "ComplexConjugateB": False, + "StochasticRounding": False, # By default, IEEE RNE rounding + # for OperationType == GEMM + "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" + "TransposeB": True, + "Batched": False, # add batching dimension + "StridedBatched": True, # use to select general batch or strided batch + "GroupedGemm": False, # use to select general batch or strided batch + # for OperationType == TensorContraction + # - Indices < NumIndicesC are Free or Batch indices and appear in C and D + # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. + # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. + # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. + # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. + # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. + # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. + # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. + # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. + "IndexAssignmentsA": [0, 2], + "IndexAssignmentsB": [1, 2], + "NumIndicesC": 2, + # use initial strides for AB. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) + "UseInitialStridesAB": False, + # use initial strides for CD. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - Additional multiply on the store address path + # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. + "UseInitialStridesCD": False, + "AllowNoFreeDims": False, # allow A or B to specify no free dims + # (if false, A and B must have at least one free dim) + # (if true, A and B must have at least one free or batch dim) + # SetConstStride* sets the specified stride in the problem. + # These no longer generate predicates - see AssertStrideEqualA/B below + # List of pairs of [index, constValue]. + # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) + # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets + # strideA for index3 to constant '1' and stride for index2 to constant '4'. + "SetConstStrideA": [], + "SetConstStrideB": [], + "SetConstStrideBias": [], + # Summation dimension indices + "MirrorDimsA": [], + "MirrorDimsB": [], + "MirrorDimsMetadata": [], + # for LD description + "NumIndicesLD": 4, + "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB + # Tile aware solution selection + "TileAwareSelection": False, + # Activation + "Activation": False, + "ActivationNoGuard": False, + # AmaxD + "OutputAmaxD": False, + # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. + "SupportUserArgs": True, + "SwizzleTensorA": False, + "SwizzleTensorB": False, +} + +# The supported typed GEMM, each entry is (Ti, To, Tc). +# DataType (Ti) = The data-type of the input matrices: A/B +# DestDataType (To) = The data-type of the output matrices: C/D +# ComputeDataType (Tc) = The data-type of computation: alpha/beta: +# Cinternal: basically should == ComputeDataType +# This is used in _checkIfSupportedGEMMType() +_validGEMMTypes = [ + ("H", "H", "H"), + ("S", "S", "S"), + ("D", "D", "D"), + ("C", "C", "C"), + ("Z", "Z", "Z"), + ("H", "H", "S"), + ("H", "S", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I8", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("B8", "B", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), + ("B8F8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), # F8 NANOO + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("B8N", "B", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), + ("B8F8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + +# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: +# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. +# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. +_HPATypes = [ + ("H", "S", "S"), + ("H", "H", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + +class ProblemType(Mapping): + ######################################## + + @classmethod + def FromDefaultConfig(printIndexAssignmentInfo: bool): + return ProblemType(_defaultProblemType, printIndexAssignmentInfo) + + def __init__(self, config, printIndexAssignmentInfo: bool): + self.state = {} + + for key in _defaultProblemType: + assignParameterWithDefault(self.state, key, config, _defaultProblemType) + + # adjusting all data types + if "DataType" in config: + self["DataType"] = DataType(config["DataType"]) + self["DataTypeA"] = self["DataType"] + self["DataTypeB"] = self["DataType"] + else: + raise Exception("NO data type specified") + self["DataType"] = DataType(0) + self["DataTypeA"] = DataType(0) + self["DataTypeB"] = DataType(0) + + if "DataTypeA" in config: + self["DataTypeA"] = DataType(config["DataTypeA"]) + + if "DataTypeB" in config: + self["DataTypeB"] = DataType(config["DataTypeB"]) + + if "DestDataType" in config: + self["DestDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["DestDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO dest data type or data type specified") + self["DataType"] = DataType(0) + + self["DataTypeE"] = self["DestDataType"] + if "DataTypeE" in config: + self["DataTypeE"] = DataType(config["DataTypeE"]) + + if "ComputeDataType" in config: + self["ComputeDataType"] = DataType(config["ComputeDataType"]) + else: + if "DestDataType" in config: + self["ComputeDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["ComputeDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO compute data type, or dest data type, or data type specified") + self["DataType"] = DataType(0) + + # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. + # So far we don't have to set it in config yamls + self["DataTypeAmaxD"] = self["ComputeDataType"] + if "DataTypeAmaxD" in config: + self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) + + if self["Sparse"]: + self["DataTypeMetadata"] = DataType("I8") + + if "F32XdlMathOp" in config: + self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) + else: + self["F32XdlMathOp"] = DataType(0) + + # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. + if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: + print2("DataType == i8 and HPA == True; setting compute data type to int32") + self["ComputeDataType"] = DataType('i') + + if self["OperationType"] == "GEMM": + self._checkIfSupportedGEMMType() + self.initGEMM() + else: + raise Exception("Unsupported OperationType = %s" % self["OperationType"]) + + self.state["AssignedDerivedParameters"] = False + ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) + + for tc in ('A', 'B'): + for sc in self["SetConstStride%s"%tc] : + (anchorDim, stride) = sc[:2] + if anchorDim not in self.state["IndexAssignments%s"%tc]: + raise Exception("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) + + # Bias + # If compute data type is not equal to dest data type, tensile will run conversion kernel. + # In this case we don't need to apply bias in beta only kernel. + if "UseBias" in config: + if self["ComputeDataType"] != self["DestDataType"]: + self["BetaOnlyUseBias"] = False + else: + self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False + if "BiasDataTypeList" in config: + self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] + self["BiasDataTypeList"].sort() # Make name unique + else: + self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) + else: + self["BetaOnlyUseBias"] = False + self["BiasDataTypeList"] = [] + + # Activation + # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. + # Otherwise, ActivationType will be set to 'none'. + if "Activation" in config: + typeStr = config.get("ActivationType", 'none') + if typeStr not in ['all', 'hipblaslt_all']: + typeStr = 'none' + else: + typeStr = 'none' + self["ActivationType"] = ActivationType(typeStr) + if "ActivationComputeDataType" in config: + self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) + else: + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if self["ActivationType"] != 'none': + # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type + if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: + printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ + ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ + self["ComputeDataType"], \ + self["DestDataType"])) + self["ActivationComputeDataType"] = self["ComputeDataType"] + if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ + (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): + printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ + ActivationComputeDataType will be set to ComputeDataType automatically.") + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if "UseE" in config: + if config["UseE"]: + if self["ActivationType"] == 'none': + printWarning("Use E is disabled cause Activation is set to False.") + self["UseE"] = False + else: + self["UseE"] = config["UseE"] + else: + self["UseE"] = config["UseE"] + + if "Gradient" in config: + if config["Gradient"]: + print1(f" --ERR-- Use bias and activation: {self['UseBias']} and {self['ActivationType']}") + if (not self["UseBias"]) and self["ActivationType"] == 'none': + printWarning("Gradient is disabled cause bias and activation are both disabled.") + self["Gradient"] = False + if self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is enabled cause Activation is enabled.") + self["UseE"] = True + elif self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is disabled cause Activation is disabled.") + self["UseE"] = False + # if self["UseScaleAlphaVec"]: + # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") + # self["UseScaleAlphaVec"] = False + self["Gradient"] = config["Gradient"] + + # Need gradient info + biasSrcList = ["A", "B", "D"] + if "BiasSrc" in config: + if not self["Gradient"] and config["BiasSrc"] != "D": + printWarning("BiasSrc is set to D cause Gradient is disabled.") + self["BiasSrc"] = "D" + elif self["Gradient"]: + # # Currently only supports D :) + # if config["BiasSrc"] != "D": + # raise Exception("BiasSrc currently only supports D.") + if config["BiasSrc"] not in biasSrcList: + raise Exception("BiasSrc only supports A, B, D.") + + if "ActivationNoGuard" in config: + self["ActivationNoGuard"] = config["ActivationNoGuard"] + if self["ActivationNoGuard"]: + if self["ActivationType"] == 'none': + printWarning("ActivationNoGuard is set to False cause Acivation is off.") + self["ActivationNoGuard"] = False + if (not self["Gradient"]): + printWarning("ActivationNoGuard is set to False cause Gradient is off.") + self["ActivationNoGuard"] = False + + ################################################################################ + # Function checkIfSupportedGEMMType: + # Assures 3 data-types are valid, supported and well-assigned + # See the discussion in ValidParameters.py for validGEMMTypes + ################################################################################ + def _checkIfSupportedGEMMType(self): + inType = self["DataType"] + outType = self["DestDataType"] + computeType = self["ComputeDataType"] + + gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) + if gemmType not in _validGEMMTypes: + raise Exception("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0], gemmType[1], gemmType[2])) + + ######################################## + def initGEMM(self): + sumIdx = 3 if self["Batched"] else 2 + self["IndexAssignmentsA"] = [0, sumIdx] # N + self["IndexAssignmentsB"] = [sumIdx, 1] # N + if self.state["Sparse"] == 2: + self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) + else: + self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) + if self["TransposeA"]: + self["IndexAssignmentsA"] = [sumIdx, 0] # T + if self["TransposeB"]: + self["IndexAssignmentsB"] = [1, sumIdx] # T + if self["Batched"]: + self["IndexAssignmentsA"].append(2) + self["IndexAssignmentsB"].append(2) + self["IndexAssignmentsMetadata"].append(2) + self["NumIndicesC"] = 3 + else: + self["NumIndicesC"] = 2 + + self["NumIndicesLD"] = 4 + self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 + for i in range(1, len(self["IndexAssignmentsLD"])): + self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 + + ######################################## + def isGEMM(self): + return self.operationType == 0 + + ######################################## + # determine d0, d1, dU + @staticmethod + def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): + if "AssignedDerivedParameters" in state: + if state["AssignedDerivedParameters"]: + return + state["AssignedDerivedParameters"] = False + + state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ + max(state["IndexAssignmentsB"])+1) + + # determine num free, batch + state["IndicesFree"] = [] + state["IndicesBatch"] = [] + state["IndicesSummation"] = [] + + for i in range(0, state["NumIndicesC"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesBatch"].append(i) + + elif inA or inB: + state["IndicesFree"].append(i) + else: + raise Exception("invalid index %u (inC but not (inA or inB))" % i) + + # determine num summation + for i in range(state["NumIndicesC"], state["TotalIndices"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesSummation"].append(i) + else: + raise Exception("invalid index %u (expected summation but not (inA and inB))" % i) + # print index assignments + if printIndexAssignmentInfo: + print("IndicesFree: %s" % state["IndicesFree"]) + print("IndicesBatch: %s" % state["IndicesBatch"]) + print("IndicesSum: %s" % state["IndicesSummation"]) + print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) + print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) + print("NumIndicesC: %s" % state["NumIndicesC"]) + + for k in ('IndexAssignmentsA','IndexAssignmentsB'): + if len(state[k]) != len(set(state[k])): + raise Exception("duplicate index in %s=%s"% (k,state[k])) + + state["NumIndicesFree"] = len(state["IndicesFree"]) + state["NumIndicesBatch"] = len(state["IndicesBatch"]) + state["NumIndicesSummation"] = len(state["IndicesSummation"]) + if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : + raise Exception("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) + + # by default, unroll index will be the last/inner summation index + state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] + for i in range(0, len(state["IndexAssignmentsA"])): + if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: + state["IndexUnrollA"] = i + break + for i in range(0, len(state["IndexAssignmentsB"])): + if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: + state["IndexUnrollB"] = i + break + for i in range(0, len(state["IndexAssignmentsMetadata"])): + if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: + state["IndexUnrollM"] = i + break + #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) + #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) + + # assign d0, d1 + if state["AllowNoFreeDims"]: + dimList = state["IndicesFree"] + state["IndicesBatch"] + else: + dimList = state["IndicesFree"] + state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] + state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] + #print2("Index01A: %u" % state["Index01A"]) + #print2("Index01B: %u" % state["Index01B"]) + # Store code is optimized for 0 as the fastest-moving in memory + # whichever has lower stride in C (lower value), is 0, other is 1 + if state["Index01A"] < state["Index01B"]: + state["Index0"] = state["Index01A"] + state["Index1"] = state["Index01B"] + state["Tensor0"] = 0 + state["Tensor1"] = 1 + state["TileA"] = 0 + state["TileB"] = 1 + else: + state["Index0"] = state["Index01B"] + state["Index1"] = state["Index01A"] + state["Tensor0"] = 1 + state["Tensor1"] = 0 + state["TileA"] = 1 + state["TileB"] = 0 + + # generalize transpose + strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) + strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) + unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) + unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) + state["TLUA"] = strideIdxA < unrollIdxA + state["TLUB"] = strideIdxB < unrollIdxB + #state["TLUB"] = True # hack + + if printIndexAssignmentInfo: + print("TLUA: %s (stridePosA(%d) 1 : + name += "_FD%s"%("N" if factorDim == 2 else "MN") + + if self["UseE"]: + if self["Gradient"]: + name += "_Grad%s"%self["DataTypeE"].toChar() + else: + name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types + if self["OutputAmaxD"]: + name += "_AmaxD" + if self["Sparse"]: + if self["Sparse"] == 2: + name += "_SPB" + else: + name += "_SPA" + + # precision and other + # name += "_SB" if self["StridedBatched"] else "_GB" + if self["GroupedGemm"]: + name += "_GG" + else: + name += "" if self["StridedBatched"] else "_GB" # legacy + + # Activation Naming + if self["ActivationType"] != 'none': + if self["ActivationType"] == 'all': + name += "_A" + elif self["ActivationType"] == 'hipblaslt_all': + name += "_HA" + else: + name += "_%s"%str(self["ActivationType"]).upper() + name += self["ActivationComputeDataType"].toChar() + if self["ActivationNoGuard"]: name += "NG" + + if self["UseScaleAB"] == "Scalar": name += "_SAB" + elif self["UseScaleAB"] == "Vector": name += "_SABV" + if self["UseScaleCD"]: name += "_SCD" + if self["UseScaleAlphaVec"]: name += "_SAV" + + if self["SupportUserArgs"]: name += "_UserArgs" + + return name + + def keys(self): + return list(self.state.keys()) + def __len__(self): + return len(self.state) + def __iter__(self): + return iter(self.state) + def __getitem__(self, key): + return self.state[key] + def __setitem__(self, key, value): + self.state[key] = value + def __repr__(self): + return self.__str__() + def getAttributes(self): + return self.state + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): + return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + + def get(self, key, default=None): + try: + return self.state[key] + except: + return default + +################################################################################ +# Bias Type +################################################################################ + +def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: + bList = [] + for d in ["DataType", "ComputeDataType", "DestDataType"]: + dtype = DataType(problem[d]) + # filter out int8, because it is not supported by bias datatype + # TODO + if not dtype.isInt8(): + bList.append(dtype) + + biasDataTypeList = list(set(bList)) + biasDataTypeList.sort() # Make name unique + return biasDataTypeList + diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index ed36dca0f8..8024ba26db 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -44,7 +44,7 @@ roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ DepthUConfig from Tensile.SolutionStructs.Naming import getNameFull -from Tensile.ProblemType import ProblemType +from Tensile.SolutionStructs.Problem import ProblemType from Tensile.Toolchain.Component import Assembler from .Utilities import reject diff --git a/tensilelite/Tensile/SolutionStructs/__init__.py b/tensilelite/Tensile/SolutionStructs/__init__.py index 8c116b21f9..cd9b96908c 100644 --- a/tensilelite/Tensile/SolutionStructs/__init__.py +++ b/tensilelite/Tensile/SolutionStructs/__init__.py @@ -1,3 +1,4 @@ +from .Naming import * from .Solution import * -from .Utilities import * from .Problem import * +from .Utilities import * diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py index 51cfdf41cf..f5eb55bbaa 100644 --- a/tensilelite/Tensile/TensileClientConfig.py +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -25,8 +25,7 @@ from . import ClientWriter from . import LibraryIO from .Contractions import ProblemType as ContractionsProblemType -from .SolutionStructs import ProblemSizes -from Tensile.ProblemType import ProblemType +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizes from .Common import globalParameters, print1, printExit, printWarning, assignGlobalParameters, \ restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters diff --git a/tensilelite/Tensile/TensileLogic/ParseArguments.py b/tensilelite/Tensile/TensileLogic/ParseArguments.py index c9db912875..70a1e03915 100644 --- a/tensilelite/Tensile/TensileLogic/ParseArguments.py +++ b/tensilelite/Tensile/TensileLogic/ParseArguments.py @@ -23,7 +23,6 @@ ################################################################################ from argparse import ArgumentParser -from typing import Any, Dict from Tensile.Toolchain.Validators import ToolchainDefaults @@ -41,6 +40,7 @@ def parseArguments(): argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") + argParser.add_argument("-v", "--verbose", dest="Verbose", type=int, default=1, choices=[0, 1, 2, 3], help="Set print level with ``--v 2``.") argParser.add_argument( "--jobs", "-j", diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index 8f023b054c..c220571e9c 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -23,27 +23,29 @@ ################################################################################ +import yaml import functools from pathlib import Path from multiprocessing import Pool from typing import List, Dict from Tensile.Common import ( - globalParameters, assignGlobalParameters, ParallelMap2, print1, - printWarning, makeIsaInfoMap, SUPPORTED_ISA, - gfxToIsa, IsaVersion, IsaInfo, + verbosity ) +from Tensile.Common.GlobalParameters import globalParameters + from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig -from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters +from Tensile.SolutionStructs import matrixInstructionToMIParameters +from Tensile import CUSTOM_KERNEL_PATH from .ParseArguments import parseArguments from .ValidMatrixInstruction import validateMatrixInstruction @@ -56,29 +58,99 @@ def getParams(isaInfoMap, cxxCompiler): return gp -def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict | None: +def handleCustomKernel(sol: dict, isaInfoMap: dict): if not isCustomKernelConfig(sol): return None name = sol["CustomKernelName"] - print1(f">> Custom kernel: {name}") - custom = getCustomKernelConfig(name, {}) + dir = CUSTOM_KERNEL_PATH + # dir = str(Path(CUSTOM_KERNEL_PATH)/".."/"NEWCustomKernels") + custom = getCustomKernelConfig(name, {}, dir) sol.update(custom) mi = sol["MatrixInstruction"] - if len(mi) != 9: - printWarning(f"Custom kernel {name} has MI length {len(mi)}, expected 9.") + print1(f">> FOUND Custom kernel: {name} with MI {mi}") + + if len(mi) == 4: + print1(f">> --DBG-- -> Success, no need to convert, has MI length {len(mi)}\n---") + return None - isa = sol["ISA"] + isa = IsaVersion(*sol["ISA"]) wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] workgroup = sol["WorkGroup"] miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + + ## Experimental custom kernel config checker code + import pprint + yamlstring = convert_pformat_to_condensed(pprint.pformat(miParams)) + print(yamlstring) + + ## end of experimetnal section + + # try: + # miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + # except Exception as e: + # printWarning(f"Custom kernel {name} failed to convert MI to parameters: {e}") + # return None sol.update(miParams) - return sol + return sol, yamlstring + + +def convert_pformat_to_condensed(pformat_str): + # Remove the dictionary braces and split into lines + lines = pformat_str.strip('{}').split('\n') + + # Initialize an empty list to store the formatted lines + formatted_lines = [] + + # Iterate over each line + for line in lines: + # Remove leading and trailing whitespace + line = line.strip() + + # Remove the 'ISA' line as it is not needed in the output + if line.startswith("'ISA'"): + continue + + line = line.replace("'", "") + line = line[:-1] if line.endswith(',') else line + + + line = ' ' + line + # Append the formatted line to the list + formatted_lines.append(line) + + # Join the formatted lines into a single string + result = '\n'.join(formatted_lines) + + return result + +def replace_line_in_file(file_path, search_string, replacement_string): + """ + Replaces a line in a file that matches the search_string with the replacement_string. + + Args: + file_path (str): The path to the file. + search_string (str): The string to search for in the file. + replacement_string (str): The multi-line string to replace the matching line with. + """ + # Read the file contents + with open(file_path, 'r') as file: + lines = file.readlines() + + # Open the file in write mode to overwrite the contents + with open(file_path, 'w') as file: + for line in lines: + if search_string in line: + # Replace the matching line with the replacement string + file.write(replacement_string + '\n') + else: + # Write the original line + file.write(line) def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): """ @@ -98,19 +170,27 @@ def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List if "Experimental" in file.parts: return keep, total + solutions = readYAML(file)[5] # Solutions are the 5th index - print1(f">> {file.relative_to(logicPath)}") + # print1(f">> {file.relative_to(logicPath)}") for s in solutions: s = handleCustomKernel(s, isaInfoMap) if s is None: continue + sol, replacement_string = s + customfile = Path(CUSTOM_KERNEL_PATH) / (sol["CustomKernelName"] + ".s") + print1(f"## UPDATING {customfile}") + replace_line_in_file(customfile, " MatrixInstruction:", replacement_string) + + + print1(f"## Custom kernel {file.relative_to(logicPath)}") if all( [ - validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), - validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), + validateMatrixInstruction(sol, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(sol, isaInfoMap, file.relative_to(logicPath)), ] ): keep += 1 @@ -124,6 +204,9 @@ def main(): print1("No checks specified. Exiting.") exit(0) + global verbosity + verbosity = args.Verbose + jobs = int(args.Jobs) cxxCompiler = validateToolchain(args.CxxCompiler) @@ -131,25 +214,29 @@ def main(): assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) logicPath = Path(args.LogicPath) - pattern = "**/*.yaml" - files = list(logicPath.glob(pattern)) + if logicPath.is_file() and logicPath.suffix == ".yaml": + files = [logicPath] + else: + pattern = "**/*.yaml" + files = list(logicPath.glob(pattern)) - batchSize = len(files) // jobs + batchSize = len(files) // min(len(files), jobs) batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) fn = functools.partial(runChecks, logicPath, isaInfoMap) keep, total = 0, 0 - with Pool(processes=jobs) as pool: - results = pool.map_async(fn, batches) - - # TIP: This is how to use joblib. Leave for reference. - # for _keep, _total in ParallelMap2( - # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" - # ): - - for _keep, _total in results.get(): - keep += _keep - total += _total + # with Pool(processes=jobs) as pool: + # results = pool.map_async(fn, batches) + + # # TIP: This is how to use joblib. Leave for reference. + results = ParallelMap2( + fn, batches, multiArg=False, procs=jobs, return_as="list" + ) + + for _keep, _total in results: + # for _keep, _total in results.get(): + keep += _keep + total += _total rejects = total - keep print(f"Total {total} solutions") diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index fe3a0c9cfa..b11351bf4b 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -51,14 +51,13 @@ - If empty, do not use these instructions """ -import math from typing import Dict from pathlib import Path -from inspect import currentframe, getframeinfo from Tensile.SolutionStructs import reject -from Tensile.Common import IsaVersion, IsaInfo, print1 +from Tensile.Common import IsaVersion, IsaInfo, print1, elineno from Tensile.Common.Architectures import SUPPORTED_ISA +from Tensile.Common.ValidParameters import validMatrixInstructions, validMFMA, validWMMA, validSMFMA from Tensile.TensileInstructions.DataType import DataType from .Utilities import elineno @@ -68,114 +67,8 @@ MI_ENABLED_KEY: str = "EnableMatrixInstruction" -validMFMA = {} -validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] -validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] -validMFMA["4xi8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], - [32, 32, 16, 1], - [16, 16, 32, 1], -] -validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] -validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["C"] = validMFMA["S"] -validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], -] + [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] -validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8"] = validMFMA["F8"] -validMFMA["F8B8"] = validMFMA["F8"] -validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8N"] = validMFMA["F8N"] -validMFMA["F8B8N"] = validMFMA["F8N"] -validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [ - [16, 16, 16, 1], -] -validTT = 32 -validMFMA["_format9"] = [] - -for MFMA in [ - validMFMA["H"], - validMFMA["S"], - validMFMA["B"], - validMFMA["D"], - validMFMA["X"], - validMFMA["F8N"], - validWMMA, -]: - for MI in MFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validMatrixInstructions = ( - [[], [-1]] - + validMFMA["H"] - + validMFMA["S"] - + validMFMA["B"] - + validMFMA["D"] - + validMFMA["B1k"] - + validMFMA["X"] -) -validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] - -validSMFMA = {} -validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8"] = validSMFMA["F8"] -validSMFMA["F8B8"] = validSMFMA["F8"] -validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8N"] = validSMFMA["F8N"] -validSMFMA["F8B8N"] = validSMFMA["F8N"] -validSMFMA["B8F8N"] = validSMFMA["F8N"] -validSMFMA["_format9"] = [] -for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validSMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = ( - validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] -) - - -def elineno(): - """ - Return the file name and line number of the caller. - """ - frame = getframeinfo(currentframe().f_back) - return f"{Path(frame.filename).name}:{frame.lineno}" - - def validateMatrixInstruction( - solution: dict, isaInfoMap: Dict[str, IsaInfo], filepath: Path + solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: Path ) -> bool: """ Validates the matrix instruction configured in the given solution. @@ -241,6 +134,11 @@ def validateMIParameters( assert miEnabled == False, elineno() return + assert solution["MatrixInstM"] == mi4[0] + assert solution["MatrixInstN"] == mi4[1] + assert solution["MatrixInstK"] == mi4[2] + assert solution["MatrixInstB"] == mi4[3] + assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] @@ -283,7 +181,7 @@ def validateMIParameters( else: return not reject( solution, - printSolutionRejectionReason, + True,#printSolutionRejectionReason, f"Invalid MFMA BFloat16 configuration: {solution}", ) elif hasWMMA and (not mi4 in validWMMA): @@ -296,10 +194,6 @@ def validateMIParameters( solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" ) - if (not hasMFMA) and hasWMMA: - if isa[0] == 10 or isa[0] == 11: - assert miInputPerThread == mi4[2], elineno() - # Check MIBlock assert miBlock[0] == mi4[0], elineno() assert miBlock[1] == mi4[1], elineno() @@ -319,6 +213,10 @@ def validateMIParameters( # Check MIInputPerThread miInputPerThread = solution["MIInputPerThread"] + if (not hasMFMA) and hasWMMA: + if isa[0] == 10 or isa[0] == 11: + assert miInputPerThread == mi4[2], elineno() + # If Navi architecture, the input per thread is different if IsaVersion(10, 0, 0) <= isa <= IsaVersion(11, 0, 2): assert miInputPerThread == mi4[2], elineno() diff --git a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py index 4761584d6a..da4dca47e0 100644 --- a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py +++ b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py @@ -31,18 +31,8 @@ from typing import Dict -from Tensile.Common import IsaVersion, IsaInfo - -from .Utilities import elineno - -validWorkGroups = [] -for numThreads in range(32, 1025, 32): - for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: - for sg0 in range(1, numThreads // nsg + 1): - sg1 = numThreads // nsg // sg0 - if sg0 * sg1 * nsg == numThreads: - workGroup = [sg0, sg1, nsg] - validWorkGroups.append(workGroup) +from Tensile.Common import IsaVersion, IsaInfo, elineno +from Tensile.Common.ValidParameters import validWorkGroups def validateWorkGroup(solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: str): diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py b/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py deleted file mode 100644 index 4d91143379..0000000000 --- a/tensilelite/Tensile/TensileLogicCUSTOM/Utilities.py +++ /dev/null @@ -1,34 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -from pathlib import Path -from inspect import currentframe, getframeinfo - - -def elineno(): - """ - Return the file name and line number of the caller. - """ - frame = getframeinfo(currentframe().f_back) - return f"{Path(frame.filename).name}:{frame.lineno}" diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py b/tensilelite/Tensile/TensileLogicOLD/ParseArguments.py similarity index 94% rename from tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py rename to tensilelite/Tensile/TensileLogicOLD/ParseArguments.py index 015485b2c8..c9db912875 100644 --- a/tensilelite/Tensile/TensileLogicCUSTOM/ParseArguments.py +++ b/tensilelite/Tensile/TensileLogicOLD/ParseArguments.py @@ -41,7 +41,6 @@ def parseArguments(): argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") - argParser.add_argument("-v", "--verbose", dest="Verbose", type=int, default=1, choices=[0, 1, 2, 3], help="Set print level with ``--v 2``.") argParser.add_argument( "--jobs", "-j", diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/Run.py b/tensilelite/Tensile/TensileLogicOLD/Run.py similarity index 51% rename from tensilelite/Tensile/TensileLogicCUSTOM/Run.py rename to tensilelite/Tensile/TensileLogicOLD/Run.py index 84abbb0942..8f023b054c 100644 --- a/tensilelite/Tensile/TensileLogicCUSTOM/Run.py +++ b/tensilelite/Tensile/TensileLogicOLD/Run.py @@ -23,7 +23,6 @@ ################################################################################ -import yaml import functools from pathlib import Path from multiprocessing import Pool @@ -34,21 +33,17 @@ assignGlobalParameters, ParallelMap2, print1, - print2, printWarning, makeIsaInfoMap, SUPPORTED_ISA, gfxToIsa, IsaVersion, IsaInfo, - verbosity ) - from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters -from Tensile import CUSTOM_KERNEL_PATH from .ParseArguments import parseArguments from .ValidMatrixInstruction import validateMatrixInstruction @@ -61,99 +56,29 @@ def getParams(isaInfoMap, cxxCompiler): return gp -def handleCustomKernel(sol: dict, isaInfoMap: dict): +def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict | None: if not isCustomKernelConfig(sol): return None name = sol["CustomKernelName"] + print1(f">> Custom kernel: {name}") - dir = CUSTOM_KERNEL_PATH - # dir = str(Path(CUSTOM_KERNEL_PATH)/".."/"NEWCustomKernels") - custom = getCustomKernelConfig(name, {}, dir) + custom = getCustomKernelConfig(name, {}) sol.update(custom) mi = sol["MatrixInstruction"] - print1(f">> FOUND Custom kernel: {name} with MI {mi}") - - if len(mi) == 4: - print1(f">> --DBG-- -> Success, no need to convert, has MI length {len(mi)}\n---") - return None + if len(mi) != 9: + printWarning(f"Custom kernel {name} has MI length {len(mi)}, expected 9.") - isa = IsaVersion(*sol["ISA"]) + isa = sol["ISA"] wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] workgroup = sol["WorkGroup"] miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - - ## Experimental custom kernel config checker code - import pprint - yamlstring = convert_pformat_to_condensed(pprint.pformat(miParams)) - print(yamlstring) - - ## end of experimetnal section - - # try: - # miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - # except Exception as e: - # printWarning(f"Custom kernel {name} failed to convert MI to parameters: {e}") - # return None sol.update(miParams) - return sol, yamlstring - - -def convert_pformat_to_condensed(pformat_str): - # Remove the dictionary braces and split into lines - lines = pformat_str.strip('{}').split('\n') - - # Initialize an empty list to store the formatted lines - formatted_lines = [] - - # Iterate over each line - for line in lines: - # Remove leading and trailing whitespace - line = line.strip() - - # Remove the 'ISA' line as it is not needed in the output - if line.startswith("'ISA'"): - continue - - line = line.replace("'", "") - line = line[:-1] if line.endswith(',') else line - - - line = ' ' + line + return sol - # Append the formatted line to the list - formatted_lines.append(line) - - # Join the formatted lines into a single string - result = '\n'.join(formatted_lines) - - return result - -def replace_line_in_file(file_path, search_string, replacement_string): - """ - Replaces a line in a file that matches the search_string with the replacement_string. - - Args: - file_path (str): The path to the file. - search_string (str): The string to search for in the file. - replacement_string (str): The multi-line string to replace the matching line with. - """ - # Read the file contents - with open(file_path, 'r') as file: - lines = file.readlines() - - # Open the file in write mode to overwrite the contents - with open(file_path, 'w') as file: - for line in lines: - if search_string in line: - # Replace the matching line with the replacement string - file.write(replacement_string + '\n') - else: - # Write the original line - file.write(line) def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): """ @@ -173,27 +98,19 @@ def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List if "Experimental" in file.parts: return keep, total - solutions = readYAML(file)[5] # Solutions are the 5th index + print1(f">> {file.relative_to(logicPath)}") - # print1(f">> {file.relative_to(logicPath)}") for s in solutions: s = handleCustomKernel(s, isaInfoMap) if s is None: continue - sol, replacement_string = s - customfile = Path(CUSTOM_KERNEL_PATH) / (sol["CustomKernelName"] + ".s") - print1(f"## UPDATING {customfile}") - replace_line_in_file(customfile, " MatrixInstruction:", replacement_string) - - - print1(f"## Custom kernel {file.relative_to(logicPath)}") if all( [ - validateMatrixInstruction(sol, isaInfoMap, file.relative_to(logicPath)), - validateWorkGroup(sol, isaInfoMap, file.relative_to(logicPath)), + validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), ] ): keep += 1 @@ -207,9 +124,6 @@ def main(): print1("No checks specified. Exiting.") exit(0) - global verbosity - verbosity = args.Verbose - jobs = int(args.Jobs) cxxCompiler = validateToolchain(args.CxxCompiler) @@ -217,29 +131,25 @@ def main(): assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) logicPath = Path(args.LogicPath) - if logicPath.is_file() and logicPath.suffix == ".yaml": - files = [logicPath] - else: - pattern = "**/*.yaml" - files = list(logicPath.glob(pattern)) + pattern = "**/*.yaml" + files = list(logicPath.glob(pattern)) - batchSize = len(files) // min(len(files), jobs) + batchSize = len(files) // jobs batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) fn = functools.partial(runChecks, logicPath, isaInfoMap) keep, total = 0, 0 - # with Pool(processes=jobs) as pool: - # results = pool.map_async(fn, batches) - - # # TIP: This is how to use joblib. Leave for reference. - results = ParallelMap2( - fn, batches, multiArg=False, procs=jobs, return_as="list" - ) - - for _keep, _total in results: - # for _keep, _total in results.get(): - keep += _keep - total += _total + with Pool(processes=jobs) as pool: + results = pool.map_async(fn, batches) + + # TIP: This is how to use joblib. Leave for reference. + # for _keep, _total in ParallelMap2( + # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" + # ): + + for _keep, _total in results.get(): + keep += _keep + total += _total rejects = total - keep print(f"Total {total} solutions") diff --git a/tensilelite/Tensile/TensileLogic/Utilities.py b/tensilelite/Tensile/TensileLogicOLD/Utilities.py similarity index 100% rename from tensilelite/Tensile/TensileLogic/Utilities.py rename to tensilelite/Tensile/TensileLogicOLD/Utilities.py diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py similarity index 98% rename from tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py rename to tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py index 0f0e4410ec..fe3a0c9cfa 100644 --- a/tensilelite/Tensile/TensileLogicCUSTOM/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py @@ -241,11 +241,6 @@ def validateMIParameters( assert miEnabled == False, elineno() return - assert solution["MatrixInstM"] == mi4[0] - assert solution["MatrixInstN"] == mi4[1] - assert solution["MatrixInstK"] == mi4[2] - assert solution["MatrixInstB"] == mi4[3] - assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] @@ -288,7 +283,7 @@ def validateMIParameters( else: return not reject( solution, - True,#printSolutionRejectionReason, + printSolutionRejectionReason, f"Invalid MFMA BFloat16 configuration: {solution}", ) elif hasWMMA and (not mi4 in validWMMA): diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py similarity index 100% rename from tensilelite/Tensile/TensileLogicCUSTOM/ValidWorkGroup.py rename to tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py diff --git a/tensilelite/Tensile/TensileLogicCUSTOM/__init__.py b/tensilelite/Tensile/TensileLogicOLD/__init__.py similarity index 100% rename from tensilelite/Tensile/TensileLogicCUSTOM/__init__.py rename to tensilelite/Tensile/TensileLogicOLD/__init__.py diff --git a/tensilelite/Tensile/bin/TensileLogic b/tensilelite/Tensile/bin/TensileLogic index 77effcf2b3..d1b70ed72c 100755 --- a/tensilelite/Tensile/bin/TensileLogic +++ b/tensilelite/Tensile/bin/TensileLogic @@ -27,7 +27,7 @@ # This script only gets called by CMake try: - from Tensile import TensileLogicCUSTOM + from Tensile import TensileLogic except ImportError: import os.path import sys @@ -35,10 +35,10 @@ except ImportError: print(parentdir) sys.path.append(parentdir) - from Tensile import TensileLogicCUSTOM + from Tensile import TensileLogic ################################################################################ # Main ################################################################################ if __name__ == "__main__": - TensileLogicCUSTOM.main() + TensileLogic.main() From 9e9cd8b641e1cd637e7daf57ef8a16c8741224a1 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:42:28 +0000 Subject: [PATCH 090/105] style: remove debug printing --- tensilelite/Tensile/BenchmarkProblems.py | 1 - tensilelite/Tensile/ProblemType.py | 1 - tensilelite/Tensile/SolutionStructs/Problem.py | 1 - tensilelite/Tensile/TensileCreateLibrary/Run.py | 1 - tensilelite/Tensile/Toolchain/Assembly.py | 1 - tensilelite/Tensile/Toolchain/Source.py | 1 - 6 files changed, 6 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 35eb2964ec..5ed02b87d4 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -92,7 +92,6 @@ def _generateForkedSolutions(problemType, constantParams, forkPermutations, asse assembler, isaInfoMap ) - print1(f" ---INF--- Solution: {solutionObject._state}") if solutionObject["Valid"]: if solutionObject not in solutionSet: solutionSet.add(solutionObject) diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py index 8a2a66631c..bfb65b6da1 100644 --- a/tensilelite/Tensile/ProblemType.py +++ b/tensilelite/Tensile/ProblemType.py @@ -410,7 +410,6 @@ def __init__(self, config, printIndexAssignmentInfo: bool): if "Gradient" in config: if config["Gradient"]: - print1(f" --ERR-- Use bias and activation: {self['UseBias']} and {self['ActivationType']}") if (not self["UseBias"]) and self["ActivationType"] == 'none': printWarning("Gradient is disabled cause bias and activation are both disabled.") self["Gradient"] = False diff --git a/tensilelite/Tensile/SolutionStructs/Problem.py b/tensilelite/Tensile/SolutionStructs/Problem.py index 8f5b34cfa8..f1c9ffba92 100644 --- a/tensilelite/Tensile/SolutionStructs/Problem.py +++ b/tensilelite/Tensile/SolutionStructs/Problem.py @@ -752,7 +752,6 @@ def __init__(self, config, printIndexAssignmentInfo: bool): if "Gradient" in config: if config["Gradient"]: - print1(f" --ERR-- Use bias and activation: {self['UseBias']} and {self['ActivationType']}") if (not self["UseBias"]) and self["ActivationType"] == 'none': printWarning("Gradient is disabled cause bias and activation are both disabled.") self["Gradient"] = False diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 1b776112fa..72e3ba87e0 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -285,7 +285,6 @@ def assemble(ret): assemblyTmpPath, compress, ) - print1(f" ---DBG--- Running buildSourceCodeObjectFiles cmdLineArchs: {cmdlineArchs}") buildSourceCodeObjectFiles( srcToolchain.compiler, srcToolchain.bundler, diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index a714d720e5..1c38e94e79 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -116,7 +116,6 @@ def buildAssemblyCodeObjectFiles( if coName: coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (kernel["BaseName"] + extObj))) - print1(f" ---DBG--- CodeObjectFileMap: {coFileMap}, compress: {compress}") for coFileRaw, objFiles in coFileMap.items(): objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) linker(objFiles, str(coFileRaw)) diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 7a50e087cf..3cd54c5b35 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -58,7 +58,6 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa """ coPath = None buildPath = Path(buildPath) - print1(f" ---DBG--- target: {target}, base: {base}, buildPath: {buildPath}, arch: {arch}") if "TensileLibrary" in base and "fallback" in base: coPath = buildPath / "{0}_{1}.hsaco.raw".format(base, arch) elif "TensileLibrary" in base: From c4a29562cf8550ef63e20ea5af2de5ba9fb93bb7 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:47:13 +0000 Subject: [PATCH 091/105] fix: problem type import and more debugging --- .../Tensile/Components/ComputeStoreVgprs.py | 1 - tensilelite/Tensile/Contractions.py | 4 +- tensilelite/Tensile/ProblemType.py | 771 ------------------ .../Tensile/SolutionStructs/Solution.py | 3 - 4 files changed, 2 insertions(+), 777 deletions(-) delete mode 100644 tensilelite/Tensile/ProblemType.py diff --git a/tensilelite/Tensile/Components/ComputeStoreVgprs.py b/tensilelite/Tensile/Components/ComputeStoreVgprs.py index bd2651715c..cda15cc55c 100644 --- a/tensilelite/Tensile/Components/ComputeStoreVgprs.py +++ b/tensilelite/Tensile/Components/ComputeStoreVgprs.py @@ -186,7 +186,6 @@ def __call__(self, writer, kernel): strideD1 = "StrideD%s" % (writer.states.indexChars[packedC1[0]]) module.add(VMulLOU32(dst=vgpr(writer.vgprs.cinRowPtr), src0=vgpr(lsuTid1), src1=sgpr(strideC1), comment=" offset 1")) module.add(VMulLOU32(dst=vgpr(writer.vgprs.coutRowPtrD), src0=vgpr(lsuTid1), src1=sgpr(strideD1), comment=" offset 1")) - printWarning(f" --ERR-- problem: {kernel['ProblemType']}, GlobalSplitU: {kernel['GlobalSplitU']}, UseE: {kernel['ProblemType']['UseE']}") if kernel["ProblemType"]["UseE"] and (kernel["GlobalSplitU"] == 1): module.add(VMovB32(dst=vgpr(writer.vgprs.coutRowPtrE), src=vgpr(lsuTid1), comment=" save offset 1 for E")) if writer.vgprs.coutRowPtrBias != -1: diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index e9d3a293d8..f6b41c1234 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -28,10 +28,10 @@ from .TensileInstructions import DataType from . import Hardware from . import Properties -from .SolutionStructs import Solution as OriginalSolution +from Tensile.SolutionStructs import Solution as OriginalSolution from .Common import gfxToIsa, internalParameters, state, state_key_ordering, \ IsaInfo, DepthUConfig -from Tensile.ProblemType import getBiasDataTypeListDefault +from Tensile.SolutionStructs.Problem import getBiasDataTypeListDefault from Tensile.Toolchain.Component import Assembler MIN_K_FOR_GSU = 32 diff --git a/tensilelite/Tensile/ProblemType.py b/tensilelite/Tensile/ProblemType.py deleted file mode 100644 index bfb65b6da1..0000000000 --- a/tensilelite/Tensile/ProblemType.py +++ /dev/null @@ -1,771 +0,0 @@ -################################################################################ -# -# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -from collections.abc import Mapping -from typing import List - -from Tensile.Activation import ActivationType -from Tensile.TensileInstructions.DataType import DataType - -from Tensile.Common.Constants import INDEX_CHARS -from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print1 - - -################################################################################ -# ProblemType -# name of solution should begin with name of problemType, and arguments can be listed out explicitly - -################################################################################ -# Default Problem Type -################################################################################ -_defaultProblemType = { - # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users - # =TensorContraction requires specifying - "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights - "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. - # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) - "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter - "UseE": False, # =True use output E to output gemm results before activation - "Gradient": False, # =True set globalWriteElements to gradient mode - "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction - "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. - "UseScaleAB": "", # Support "", "Scalar", and "Vector" - "UseScaleCD": False, # =True use scaleC, scaleD - "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction - "HighPrecisionAccumulate": False, # f32 += f16*f16 - "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. - "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case - "ComplexConjugateB": False, - "StochasticRounding": False, # By default, IEEE RNE rounding - # for OperationType == GEMM - "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" - "TransposeB": True, - "Batched": False, # add batching dimension - "StridedBatched": True, # use to select general batch or strided batch - "GroupedGemm": False, # use to select general batch or strided batch - # for OperationType == TensorContraction - # - Indices < NumIndicesC are Free or Batch indices and appear in C and D - # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. - # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. - # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. - # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. - # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. - # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. - # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. - # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. - "IndexAssignmentsA": [0, 2], - "IndexAssignmentsB": [1, 2], - "NumIndicesC": 2, - # use initial strides for AB. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) - "UseInitialStridesAB": False, - # use initial strides for CD. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - Additional multiply on the store address path - # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. - "UseInitialStridesCD": False, - "AllowNoFreeDims": False, # allow A or B to specify no free dims - # (if false, A and B must have at least one free dim) - # (if true, A and B must have at least one free or batch dim) - # SetConstStride* sets the specified stride in the problem. - # These no longer generate predicates - see AssertStrideEqualA/B below - # List of pairs of [index, constValue]. - # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) - # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets - # strideA for index3 to constant '1' and stride for index2 to constant '4'. - "SetConstStrideA": [], - "SetConstStrideB": [], - "SetConstStrideBias": [], - # Summation dimension indices - "MirrorDimsA": [], - "MirrorDimsB": [], - "MirrorDimsMetadata": [], - # for LD description - "NumIndicesLD": 4, - "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB - # Tile aware solution selection - "TileAwareSelection": False, - # Activation - "Activation": False, - "ActivationNoGuard": False, - # AmaxD - "OutputAmaxD": False, - # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. - "SupportUserArgs": True, - "SwizzleTensorA": False, - "SwizzleTensorB": False, -} - -# The supported typed GEMM, each entry is (Ti, To, Tc). -# DataType (Ti) = The data-type of the input matrices: A/B -# DestDataType (To) = The data-type of the output matrices: C/D -# ComputeDataType (Tc) = The data-type of computation: alpha/beta: -# Cinternal: basically should == ComputeDataType -# This is used in _checkIfSupportedGEMMType() -_validGEMMTypes = [ - ("H", "H", "H"), - ("S", "S", "S"), - ("D", "D", "D"), - ("C", "C", "C"), - ("Z", "Z", "Z"), - ("H", "H", "S"), - ("H", "S", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I8", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("B8", "B", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), - ("B8F8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), # F8 NANOO - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("B8N", "B", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), - ("B8F8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - - -# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: -# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. -# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. -_HPATypes = [ - ("H", "S", "S"), - ("H", "H", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - - -class ProblemType(Mapping): - ######################################## - - @classmethod - def FromDefaultConfig(printIndexAssignmentInfo: bool): - return ProblemType(_defaultProblemType, printIndexAssignmentInfo) - - def __init__(self, config, printIndexAssignmentInfo: bool): - self.state = {} - - for key in _defaultProblemType: - assignParameterWithDefault(self.state, key, config, _defaultProblemType) - - # adjusting all data types - if "DataType" in config: - self["DataType"] = DataType(config["DataType"]) - self["DataTypeA"] = self["DataType"] - self["DataTypeB"] = self["DataType"] - else: - raise Exception("NO data type specified") - self["DataType"] = DataType(0) - self["DataTypeA"] = DataType(0) - self["DataTypeB"] = DataType(0) - - if "DataTypeA" in config: - self["DataTypeA"] = DataType(config["DataTypeA"]) - - if "DataTypeB" in config: - self["DataTypeB"] = DataType(config["DataTypeB"]) - - if "DestDataType" in config: - self["DestDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["DestDataType"] = DataType(config["DataType"]) - else: - raise Exception("NO dest data type or data type specified") - self["DataType"] = DataType(0) - - self["DataTypeE"] = self["DestDataType"] - if "DataTypeE" in config: - self["DataTypeE"] = DataType(config["DataTypeE"]) - - if "ComputeDataType" in config: - self["ComputeDataType"] = DataType(config["ComputeDataType"]) - else: - if "DestDataType" in config: - self["ComputeDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["ComputeDataType"] = DataType(config["DataType"]) - else: - raise Exception("NO compute data type, or dest data type, or data type specified") - self["DataType"] = DataType(0) - - # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. - # So far we don't have to set it in config yamls - self["DataTypeAmaxD"] = self["ComputeDataType"] - if "DataTypeAmaxD" in config: - self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) - - if self["Sparse"]: - self["DataTypeMetadata"] = DataType("I8") - - if "F32XdlMathOp" in config: - self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) - else: - self["F32XdlMathOp"] = DataType(0) - - # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. - if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: - print2("DataType == i8 and HPA == True; setting compute data type to int32") - self["ComputeDataType"] = DataType('i') - - if self["OperationType"] == "GEMM": - self._checkIfSupportedGEMMType() - self.initGEMM() - else: - raise Exception("Unsupported OperationType = %s" % self["OperationType"]) - - self.state["AssignedDerivedParameters"] = False - ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) - - for tc in ('A', 'B'): - for sc in self["SetConstStride%s"%tc] : - (anchorDim, stride) = sc[:2] - if anchorDim not in self.state["IndexAssignments%s"%tc]: - raise Exception("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) - - # Bias - # If compute data type is not equal to dest data type, tensile will run conversion kernel. - # In this case we don't need to apply bias in beta only kernel. - if "UseBias" in config: - if self["ComputeDataType"] != self["DestDataType"]: - self["BetaOnlyUseBias"] = False - else: - self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False - if "BiasDataTypeList" in config: - self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] - self["BiasDataTypeList"].sort() # Make name unique - else: - self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) - else: - self["BetaOnlyUseBias"] = False - self["BiasDataTypeList"] = [] - - # Activation - # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. - # Otherwise, ActivationType will be set to 'none'. - if "Activation" in config: - typeStr = config.get("ActivationType", 'none') - if typeStr not in ['all', 'hipblaslt_all']: - typeStr = 'none' - else: - typeStr = 'none' - self["ActivationType"] = ActivationType(typeStr) - if "ActivationComputeDataType" in config: - self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) - else: - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if self["ActivationType"] != 'none': - # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type - if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: - printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ - ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ - self["ComputeDataType"], \ - self["DestDataType"])) - self["ActivationComputeDataType"] = self["ComputeDataType"] - if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ - (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): - printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ - ActivationComputeDataType will be set to ComputeDataType automatically.") - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if "UseE" in config: - if config["UseE"]: - if self["ActivationType"] == 'none': - printWarning("Use E is disabled cause Activation is set to False.") - self["UseE"] = False - else: - self["UseE"] = config["UseE"] - else: - self["UseE"] = config["UseE"] - - if "Gradient" in config: - if config["Gradient"]: - if (not self["UseBias"]) and self["ActivationType"] == 'none': - printWarning("Gradient is disabled cause bias and activation are both disabled.") - self["Gradient"] = False - if self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is enabled cause Activation is enabled.") - self["UseE"] = True - elif self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is disabled cause Activation is disabled.") - self["UseE"] = False - # if self["UseScaleAlphaVec"]: - # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") - # self["UseScaleAlphaVec"] = False - self["Gradient"] = config["Gradient"] - - # Need gradient info - biasSrcList = ["A", "B", "D"] - if "BiasSrc" in config: - if not self["Gradient"] and config["BiasSrc"] != "D": - printWarning("BiasSrc is set to D cause Gradient is disabled.") - self["BiasSrc"] = "D" - elif self["Gradient"]: - # # Currently only supports D :) - # if config["BiasSrc"] != "D": - # raise Exception("BiasSrc currently only supports D.") - if config["BiasSrc"] not in biasSrcList: - raise Exception("BiasSrc only supports A, B, D.") - - if "ActivationNoGuard" in config: - self["ActivationNoGuard"] = config["ActivationNoGuard"] - if self["ActivationNoGuard"]: - if self["ActivationType"] == 'none': - printWarning("ActivationNoGuard is set to False cause Acivation is off.") - self["ActivationNoGuard"] = False - if (not self["Gradient"]): - printWarning("ActivationNoGuard is set to False cause Gradient is off.") - self["ActivationNoGuard"] = False - - ################################################################################ - # Function checkIfSupportedGEMMType: - # Assures 3 data-types are valid, supported and well-assigned - # See the discussion in ValidParameters.py for validGEMMTypes - ################################################################################ - def _checkIfSupportedGEMMType(self): - inType = self["DataType"] - outType = self["DestDataType"] - computeType = self["ComputeDataType"] - - gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) - if gemmType not in _validGEMMTypes: - raise Exception("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0], gemmType[1], gemmType[2])) - - ######################################## - def initGEMM(self): - sumIdx = 3 if self["Batched"] else 2 - self["IndexAssignmentsA"] = [0, sumIdx] # N - self["IndexAssignmentsB"] = [sumIdx, 1] # N - if self.state["Sparse"] == 2: - self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) - else: - self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) - if self["TransposeA"]: - self["IndexAssignmentsA"] = [sumIdx, 0] # T - if self["TransposeB"]: - self["IndexAssignmentsB"] = [1, sumIdx] # T - if self["Batched"]: - self["IndexAssignmentsA"].append(2) - self["IndexAssignmentsB"].append(2) - self["IndexAssignmentsMetadata"].append(2) - self["NumIndicesC"] = 3 - else: - self["NumIndicesC"] = 2 - - self["NumIndicesLD"] = 4 - self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 - for i in range(1, len(self["IndexAssignmentsLD"])): - self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 - - ######################################## - def isGEMM(self): - return self.operationType == 0 - - ######################################## - # determine d0, d1, dU - @staticmethod - def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): - if "AssignedDerivedParameters" in state: - if state["AssignedDerivedParameters"]: - return - state["AssignedDerivedParameters"] = False - - state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ - max(state["IndexAssignmentsB"])+1) - - # determine num free, batch - state["IndicesFree"] = [] - state["IndicesBatch"] = [] - state["IndicesSummation"] = [] - - for i in range(0, state["NumIndicesC"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesBatch"].append(i) - - elif inA or inB: - state["IndicesFree"].append(i) - else: - raise Exception("invalid index %u (inC but not (inA or inB))" % i) - - # determine num summation - for i in range(state["NumIndicesC"], state["TotalIndices"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesSummation"].append(i) - else: - raise Exception("invalid index %u (expected summation but not (inA and inB))" % i) - # print index assignments - if printIndexAssignmentInfo: - print("IndicesFree: %s" % state["IndicesFree"]) - print("IndicesBatch: %s" % state["IndicesBatch"]) - print("IndicesSum: %s" % state["IndicesSummation"]) - print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) - print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) - print("NumIndicesC: %s" % state["NumIndicesC"]) - - for k in ('IndexAssignmentsA','IndexAssignmentsB'): - if len(state[k]) != len(set(state[k])): - raise Exception("duplicate index in %s=%s"% (k,state[k])) - - state["NumIndicesFree"] = len(state["IndicesFree"]) - state["NumIndicesBatch"] = len(state["IndicesBatch"]) - state["NumIndicesSummation"] = len(state["IndicesSummation"]) - if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : - raise Exception("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) - - # by default, unroll index will be the last/inner summation index - state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] - for i in range(0, len(state["IndexAssignmentsA"])): - if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: - state["IndexUnrollA"] = i - break - for i in range(0, len(state["IndexAssignmentsB"])): - if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: - state["IndexUnrollB"] = i - break - for i in range(0, len(state["IndexAssignmentsMetadata"])): - if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: - state["IndexUnrollM"] = i - break - #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) - #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) - - # assign d0, d1 - if state["AllowNoFreeDims"]: - dimList = state["IndicesFree"] + state["IndicesBatch"] - else: - dimList = state["IndicesFree"] - state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] - state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] - #print2("Index01A: %u" % state["Index01A"]) - #print2("Index01B: %u" % state["Index01B"]) - # Store code is optimized for 0 as the fastest-moving in memory - # whichever has lower stride in C (lower value), is 0, other is 1 - if state["Index01A"] < state["Index01B"]: - state["Index0"] = state["Index01A"] - state["Index1"] = state["Index01B"] - state["Tensor0"] = 0 - state["Tensor1"] = 1 - state["TileA"] = 0 - state["TileB"] = 1 - else: - state["Index0"] = state["Index01B"] - state["Index1"] = state["Index01A"] - state["Tensor0"] = 1 - state["Tensor1"] = 0 - state["TileA"] = 1 - state["TileB"] = 0 - - # generalize transpose - strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) - strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) - unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) - unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) - state["TLUA"] = strideIdxA < unrollIdxA - state["TLUB"] = strideIdxB < unrollIdxB - #state["TLUB"] = True # hack - - if printIndexAssignmentInfo: - print("TLUA: %s (stridePosA(%d) 1 : - name += "_FD%s"%("N" if factorDim == 2 else "MN") - - if self["UseE"]: - if self["Gradient"]: - name += "_Grad%s"%self["DataTypeE"].toChar() - else: - name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types - if self["OutputAmaxD"]: - name += "_AmaxD" - if self["Sparse"]: - if self["Sparse"] == 2: - name += "_SPB" - else: - name += "_SPA" - - # precision and other - # name += "_SB" if self["StridedBatched"] else "_GB" - if self["GroupedGemm"]: - name += "_GG" - else: - name += "" if self["StridedBatched"] else "_GB" # legacy - - # Activation Naming - if self["ActivationType"] != 'none': - if self["ActivationType"] == 'all': - name += "_A" - elif self["ActivationType"] == 'hipblaslt_all': - name += "_HA" - else: - name += "_%s"%str(self["ActivationType"]).upper() - name += self["ActivationComputeDataType"].toChar() - if self["ActivationNoGuard"]: name += "NG" - - if self["UseScaleAB"] == "Scalar": name += "_SAB" - elif self["UseScaleAB"] == "Vector": name += "_SABV" - if self["UseScaleCD"]: name += "_SCD" - if self["UseScaleAlphaVec"]: name += "_SAV" - - if self["SupportUserArgs"]: name += "_UserArgs" - - return name - - def keys(self): - return list(self.state.keys()) - def __len__(self): - return len(self.state) - def __iter__(self): - return iter(self.state) - def __getitem__(self, key): - return self.state[key] - def __setitem__(self, key, value): - self.state[key] = value - def __repr__(self): - return self.__str__() - def getAttributes(self): - return self.state - def __hash__(self): - return hash(str(self)) - def __eq__(self, other): - return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() - def __ne__(self, other): - result = self.__eq__(other) - if result is NotImplemented: - return result - return not result - - def get(self, key, default=None): - try: - return self.state[key] - except: - return default - -################################################################################ -# Bias Type -################################################################################ - -def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: - bList = [] - for d in ["DataType", "ComputeDataType", "DestDataType"]: - dtype = DataType(problem[d]) - # filter out int8, because it is not supported by bias datatype - # TODO - if not dtype.isInt8(): - bList.append(dtype) - - biasDataTypeList = list(set(bList)) - biasDataTypeList.sort() # Make name unique - return biasDataTypeList diff --git a/tensilelite/Tensile/SolutionStructs/Solution.py b/tensilelite/Tensile/SolutionStructs/Solution.py index 8024ba26db..ace3faa48d 100644 --- a/tensilelite/Tensile/SolutionStructs/Solution.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -514,7 +514,6 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, reject(state, printRejectionReason, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") # macro tile sizes - printWarning(f" ---ERR--- seting macro tile: {state['SubGroup0']} {state['ThreadTile0']} {state['SubGroup1']} {state['ThreadTile1']}") if "SubGroup0" in state and "ThreadTile0" in state: state["MacroTile0"] = state["SubGroup0"]*state["ThreadTile0"] if "SubGroup1" in state and "ThreadTile1" in state: @@ -523,8 +522,6 @@ def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, if state["MacroTile0"] != state["MacroTile"][0] \ or state["MacroTile1"] != state["MacroTile"][1]: reject(state, printRejectionReason, "MacroTile mismatch") - printWarning(f" ---ERR--- seting macro tile: {state['MacroTile0']} {state['MacroTile1']}") - printWarning(f" ---ERR--- global split U: {state['GlobalSplitU']}") # tail loop optimization state["tailLoopOptA"] = True From 08d9a9c867765d5b04af03611e71dab8ea5b5640 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:52:01 +0000 Subject: [PATCH 092/105] fix: bad import for elineno --- tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index b11351bf4b..ec307b3d85 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -60,8 +60,6 @@ from Tensile.Common.ValidParameters import validMatrixInstructions, validMFMA, validWMMA, validSMFMA from Tensile.TensileInstructions.DataType import DataType -from .Utilities import elineno - MI_KEY: str = "MatrixInstruction" MI_ENABLED_KEY: str = "EnableMatrixInstruction" From de315b29107ce01e0594dde88d4fd811c72ca60f Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:36:06 +0000 Subject: [PATCH 093/105] fix: 9-item to 4-item mapping for custom kernel DTVA --- ...x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s index 4f9a429c85..6d9369b256 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: False SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4, 16, 4, 1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 From 5a59fec10d06efa01532973f5dc3bc81197f2612 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:37:47 +0000 Subject: [PATCH 094/105] fix: tensile logic check --- tensilelite/Tensile/CustomKernels.py | 8 +- tensilelite/Tensile/TensileLogic/Run.py | 125 +------ .../TensileLogic/ValidMatrixInstruction.py | 9 +- .../Tensile/TensileLogicOLD/ParseArguments.py | 61 ---- tensilelite/Tensile/TensileLogicOLD/Run.py | 160 --------- .../Tensile/TensileLogicOLD/Utilities.py | 34 -- .../TensileLogicOLD/ValidMatrixInstruction.py | 337 ------------------ .../Tensile/TensileLogicOLD/ValidWorkGroup.py | 62 ---- .../Tensile/TensileLogicOLD/__init__.py | 1 - 9 files changed, 28 insertions(+), 769 deletions(-) delete mode 100644 tensilelite/Tensile/TensileLogicOLD/ParseArguments.py delete mode 100644 tensilelite/Tensile/TensileLogicOLD/Run.py delete mode 100644 tensilelite/Tensile/TensileLogicOLD/Utilities.py delete mode 100644 tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py delete mode 100644 tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py delete mode 100644 tensilelite/Tensile/TensileLogicOLD/__init__.py diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index 2022102792..9ee5afb60e 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -93,9 +93,15 @@ def getCustomKernelConfig( for key in internalSupportParams: if key not in kernelIsp: kernelIsp[key] = internalSupportParams[key] + + # validParams = validParameters.update(newMIValidParameters) + # if not validParams: + # raise RuntimeError(f"Valid parameters not loaded: {validParameters}\n{newMIValidParameters}\n{validParams}") + validParameters.update(newMIValidParameters) + for k, v in kernelConfig.items(): if k != "ProblemType": - checkParametersAreValid((k, [v]), validParameters + newMIValidParameters) + checkParametersAreValid((k, [v]), validParameters) kernelConfig["KernelLanguage"] = "Assembly" kernelConfig["CustomKernelName"] = kernelName diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index c220571e9c..faf0ae32de 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -44,7 +44,6 @@ from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig -from Tensile.SolutionStructs import matrixInstructionToMIParameters from Tensile import CUSTOM_KERNEL_PATH from .ParseArguments import parseArguments @@ -52,105 +51,23 @@ from .ValidWorkGroup import validateWorkGroup -def getParams(isaInfoMap, cxxCompiler): - gp = globalParameters - assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap, cxxCompiler) - return gp - - def handleCustomKernel(sol: dict, isaInfoMap: dict): if not isCustomKernelConfig(sol): - return None + return sol name = sol["CustomKernelName"] - dir = CUSTOM_KERNEL_PATH - # dir = str(Path(CUSTOM_KERNEL_PATH)/".."/"NEWCustomKernels") - custom = getCustomKernelConfig(name, {}, dir) - sol.update(custom) + config = getCustomKernelConfig(name, {}, dir) + sol.update(config) mi = sol["MatrixInstruction"] - print1(f">> FOUND Custom kernel: {name} with MI {mi}") - - if len(mi) == 4: - print1(f">> --DBG-- -> Success, no need to convert, has MI length {len(mi)}\n---") - return None - - isa = IsaVersion(*sol["ISA"]) - wavefrontSize = sol["WavefrontSize"] - ptype = sol["ProblemType"] - workgroup = sol["WorkGroup"] - - miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - - ## Experimental custom kernel config checker code - import pprint - yamlstring = convert_pformat_to_condensed(pprint.pformat(miParams)) - print(yamlstring) - - ## end of experimetnal section - - # try: - # miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - # except Exception as e: - # printWarning(f"Custom kernel {name} failed to convert MI to parameters: {e}") - # return None - sol.update(miParams) - return sol, yamlstring - - -def convert_pformat_to_condensed(pformat_str): - # Remove the dictionary braces and split into lines - lines = pformat_str.strip('{}').split('\n') - - # Initialize an empty list to store the formatted lines - formatted_lines = [] - - # Iterate over each line - for line in lines: - # Remove leading and trailing whitespace - line = line.strip() - - # Remove the 'ISA' line as it is not needed in the output - if line.startswith("'ISA'"): - continue - - line = line.replace("'", "") - line = line[:-1] if line.endswith(',') else line - - - line = ' ' + line + print1(f">> Found custom kernel: {name} with MI {mi}") - # Append the formatted line to the list - formatted_lines.append(line) + if not (len(mi) == 4 or len(mi) == 0): + raise ValueError(f">> Error: Custom kernels should have matrix instruction of length 4, or none at all, not length {len(mi)}\n{name}") - # Join the formatted lines into a single string - result = '\n'.join(formatted_lines) + return sol - return result - -def replace_line_in_file(file_path, search_string, replacement_string): - """ - Replaces a line in a file that matches the search_string with the replacement_string. - - Args: - file_path (str): The path to the file. - search_string (str): The string to search for in the file. - replacement_string (str): The multi-line string to replace the matching line with. - """ - # Read the file contents - with open(file_path, 'r') as file: - lines = file.readlines() - - # Open the file in write mode to overwrite the contents - with open(file_path, 'w') as file: - for line in lines: - if search_string in line: - # Replace the matching line with the replacement string - file.write(replacement_string + '\n') - else: - # Write the original line - file.write(line) def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): """ @@ -173,24 +90,14 @@ def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List solutions = readYAML(file)[5] # Solutions are the 5th index - # print1(f">> {file.relative_to(logicPath)}") + print1(f">> {file.relative_to(logicPath)}") for s in solutions: s = handleCustomKernel(s, isaInfoMap) - if s is None: - continue - - sol, replacement_string = s - customfile = Path(CUSTOM_KERNEL_PATH) / (sol["CustomKernelName"] + ".s") - print1(f"## UPDATING {customfile}") - replace_line_in_file(customfile, " MatrixInstruction:", replacement_string) - - - print1(f"## Custom kernel {file.relative_to(logicPath)}") if all( [ - validateMatrixInstruction(sol, isaInfoMap, file.relative_to(logicPath)), - validateWorkGroup(sol, isaInfoMap, file.relative_to(logicPath)), + validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), ] ): keep += 1 @@ -200,9 +107,6 @@ def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List def main(): args = parseArguments() - if not any([args.Check]): - print1("No checks specified. Exiting.") - exit(0) global verbosity verbosity = args.Verbose @@ -220,6 +124,15 @@ def main(): pattern = "**/*.yaml" files = list(logicPath.glob(pattern)) + if not any([args.Check]): + print1("No checks specified. Exiting.") + exit(0) + if len(files) == 0: + print1(f"No files found in {logicPath}") + exit(1) + + print1(f"Found {len(files)} files") + batchSize = len(files) // min(len(files), jobs) batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index ec307b3d85..7279716c6e 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -160,7 +160,6 @@ def validateMIParameters( hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] - miBlock = solution["MIBlock"] miWaveGroup = solution["MIWaveGroup"] miWaveTile = solution["MIWaveTile"] @@ -168,12 +167,8 @@ def validateMIParameters( # Check datatype if not isSparse: # If it's sparse if hasMFMA: # and it supports MFMA - if not ( - miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()] - ): # but is invalid MFMA - print1( - f"Looks like {mi4} of type {miDataType.toChar()} is not supported for MFMA {validMFMA[miDataType.toChar()]}" - ) + # but is invalid MFMA + if not miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]: if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA assert solution["MFMA_BF16_1K"], elineno() else: diff --git a/tensilelite/Tensile/TensileLogicOLD/ParseArguments.py b/tensilelite/Tensile/TensileLogicOLD/ParseArguments.py deleted file mode 100644 index c9db912875..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/ParseArguments.py +++ /dev/null @@ -1,61 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -from argparse import ArgumentParser -from typing import Any, Dict - -from Tensile.Toolchain.Validators import ToolchainDefaults - - -def parseArguments(): - """ - Returns: - A dictionary containing the keys representing options and their values. - """ - - argParser = ArgumentParser( - description="TensileValidateLogic runs critical checks to ensure the " - "integrity of the supplied logic files.", - ) - - argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") - argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") - argParser.add_argument( - "--jobs", - "-j", - dest="Jobs", - action="store", - default=48, - help="Number of worker processes to use during validation checks.", - ) - argParser.add_argument( - "--cxx-compiler", - dest="CxxCompiler", - action="store", - default=ToolchainDefaults.CXX_COMPILER, - help=f"Default: {ToolchainDefaults.CXX_COMPILER}", - ) - args = argParser.parse_args() - - return args diff --git a/tensilelite/Tensile/TensileLogicOLD/Run.py b/tensilelite/Tensile/TensileLogicOLD/Run.py deleted file mode 100644 index 8f023b054c..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/Run.py +++ /dev/null @@ -1,160 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - - -import functools -from pathlib import Path -from multiprocessing import Pool -from typing import List, Dict - -from Tensile.Common import ( - globalParameters, - assignGlobalParameters, - ParallelMap2, - print1, - printWarning, - makeIsaInfoMap, - SUPPORTED_ISA, - gfxToIsa, - IsaVersion, - IsaInfo, -) -from Tensile.LibraryIO import readYAML -from Tensile.Toolchain.Validators import validateToolchain -from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig -from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters - -from .ParseArguments import parseArguments -from .ValidMatrixInstruction import validateMatrixInstruction -from .ValidWorkGroup import validateWorkGroup - - -def getParams(isaInfoMap, cxxCompiler): - gp = globalParameters - assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap, cxxCompiler) - return gp - - -def handleCustomKernel(sol: dict, isaInfoMap: dict) -> dict | None: - if not isCustomKernelConfig(sol): - return None - - name = sol["CustomKernelName"] - print1(f">> Custom kernel: {name}") - - custom = getCustomKernelConfig(name, {}) - sol.update(custom) - - mi = sol["MatrixInstruction"] - if len(mi) != 9: - printWarning(f"Custom kernel {name} has MI length {len(mi)}, expected 9.") - - isa = sol["ISA"] - wavefrontSize = sol["WavefrontSize"] - ptype = sol["ProblemType"] - workgroup = sol["WorkGroup"] - - miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - sol.update(miParams) - return sol - - -def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): - """ - Run checks on the given files. - - Args: - logicPath: Path to the logic directory. - gp: Global parameters. - files: List of files to check. - - Returns: - Tuple of (keep, total) where keep is the number of solutions to keep and - total is the total number of solutions. - """ - keep, total = 0, 0 - for file in files: - if "Experimental" in file.parts: - return keep, total - - solutions = readYAML(file)[5] # Solutions are the 5th index - print1(f">> {file.relative_to(logicPath)}") - - for s in solutions: - s = handleCustomKernel(s, isaInfoMap) - - if s is None: - continue - - if all( - [ - validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), - validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), - ] - ): - keep += 1 - total += 1 - return keep, total - - -def main(): - args = parseArguments() - if not any([args.Check]): - print1("No checks specified. Exiting.") - exit(0) - - jobs = int(args.Jobs) - cxxCompiler = validateToolchain(args.CxxCompiler) - - isaInfoMap = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) - assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) - - logicPath = Path(args.LogicPath) - pattern = "**/*.yaml" - files = list(logicPath.glob(pattern)) - - batchSize = len(files) // jobs - batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) - - fn = functools.partial(runChecks, logicPath, isaInfoMap) - keep, total = 0, 0 - with Pool(processes=jobs) as pool: - results = pool.map_async(fn, batches) - - # TIP: This is how to use joblib. Leave for reference. - # for _keep, _total in ParallelMap2( - # fn, batches, multiArg=False, procs=jobs, return_as="generator_unordered" - # ): - - for _keep, _total in results.get(): - keep += _keep - total += _total - - rejects = total - keep - print(f"Total {total} solutions") - print(f"Keep {keep} solutions") - print(f"Reject {rejects} solutions") - - if rejects > 0: - exit(1) diff --git a/tensilelite/Tensile/TensileLogicOLD/Utilities.py b/tensilelite/Tensile/TensileLogicOLD/Utilities.py deleted file mode 100644 index 4d91143379..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/Utilities.py +++ /dev/null @@ -1,34 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -from pathlib import Path -from inspect import currentframe, getframeinfo - - -def elineno(): - """ - Return the file name and line number of the caller. - """ - frame = getframeinfo(currentframe().f_back) - return f"{Path(frame.filename).name}:{frame.lineno}" diff --git a/tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py deleted file mode 100644 index fe3a0c9cfa..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/ValidMatrixInstruction.py +++ /dev/null @@ -1,337 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -""" -ValidMatrixInstruction ---- -Format: (M x N x K x B) - XDLOPS tile definition, only valid for gfx908, gfx90a - MxNxKxB specifies matrix instruction variants - MxNxB determines the shape of the C tile each instruction worked on - K determines the unroll depth - -Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) - (Note: MxN means M-by-N in the following comments) - MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: - - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed - - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed - WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels - - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) - WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads - - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 - Putting it all together: - - [32, 32, 1, 2, 1, 4, 1, 2, 2] - ^^^^^^^^^^^^ ^ ^^^^ ^^^^ - MatrixInst BlkM WT Wave - - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile - Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used - -Notes: - - If empty, do not use these instructions -""" - -import math -from typing import Dict -from pathlib import Path -from inspect import currentframe, getframeinfo - -from Tensile.SolutionStructs import reject -from Tensile.Common import IsaVersion, IsaInfo, print1 -from Tensile.Common.Architectures import SUPPORTED_ISA -from Tensile.TensileInstructions.DataType import DataType - -from .Utilities import elineno - - -MI_KEY: str = "MatrixInstruction" -MI_ENABLED_KEY: str = "EnableMatrixInstruction" - - -validMFMA = {} -validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] -validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] -validMFMA["4xi8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], - [32, 32, 16, 1], - [16, 16, 32, 1], -] -validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] -validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["C"] = validMFMA["S"] -validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], -] + [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] -validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8"] = validMFMA["F8"] -validMFMA["F8B8"] = validMFMA["F8"] -validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8N"] = validMFMA["F8N"] -validMFMA["F8B8N"] = validMFMA["F8N"] -validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [ - [16, 16, 16, 1], -] -validTT = 32 -validMFMA["_format9"] = [] - -for MFMA in [ - validMFMA["H"], - validMFMA["S"], - validMFMA["B"], - validMFMA["D"], - validMFMA["X"], - validMFMA["F8N"], - validWMMA, -]: - for MI in MFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validMatrixInstructions = ( - [[], [-1]] - + validMFMA["H"] - + validMFMA["S"] - + validMFMA["B"] - + validMFMA["D"] - + validMFMA["B1k"] - + validMFMA["X"] -) -validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] - -validSMFMA = {} -validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8"] = validSMFMA["F8"] -validSMFMA["F8B8"] = validSMFMA["F8"] -validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8N"] = validSMFMA["F8N"] -validSMFMA["F8B8N"] = validSMFMA["F8N"] -validSMFMA["B8F8N"] = validSMFMA["F8N"] -validSMFMA["_format9"] = [] -for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validSMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = ( - validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] -) - - -def elineno(): - """ - Return the file name and line number of the caller. - """ - frame = getframeinfo(currentframe().f_back) - return f"{Path(frame.filename).name}:{frame.lineno}" - - -def validateMatrixInstruction( - solution: dict, isaInfoMap: Dict[str, IsaInfo], filepath: Path -) -> bool: - """ - Validates the matrix instruction configured in the given solution. - - The function performs the following checks: - - Ensures that the solution contains the required keys for matrix instruction support. - - Ensures that the matrix instruction is not empty when it is enabled. - - Validates that the matrix instruction is in the list of valid matrix instructions. - - If the matrix instruction has 9 elements, it performs detailed validation checks: - - Validates the work group dimensions. - - Checks if the matrix instruction is supported by the assembler capabilities (MFMA or WMMA). - - Validates the input per thread for sparse and non-sparse configurations. - - Validates the matrix instruction block, wave group, and wave tile dimensions. - - If the matrix instruction has 4 elements, it ensures that matrix instructions are enabled. - - If the matrix instruction is empty, it ensures that matrix instructions are disabled. - - Args: - solution: The solution to validate. - filepath: The path to the file containing the solution. - params: The global parameters for the solution. - - Raises: - AssertionError: If any of the validation checks fail. - """ - try: - validateMIParameters(solution, isaInfoMap) - assert solution["Valid"], f"Solution was rejected: {elineno()}" - return True - except AssertionError as e: - print( - f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" - ) - return False - - -def validateMIParameters( - solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], printSolutionRejectionReason: bool = True -): - assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" - assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" - assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), ( - elineno() + ": MI empty but enabled" - ) - - isa = IsaVersion(*solution["ISA"]) - assert isa in SUPPORTED_ISA, elineno() + ": Unsupported ISA: " + str(isa) - # TODO: Temporary until all 940/941 ISAs are removed - if (9, 4, 0) <= isa <= (9, 4, 1): - isa = (9, 4, 2) - - ptype = solution["ProblemType"] - isSparse = ptype.get("Sparse", 0) - miDataType = DataType( - ptype["DataType"] - if not solution.get("EnableF32XdlMathOp", False) - else ptype["F32XdlMathOp"] - ) - - mi4 = solution[MI_KEY] - miEnabled = solution[MI_ENABLED_KEY] - assert len(mi4) == 4 or len(mi4) == 0, elineno() + ": MI length not 4 or 0" - if len(mi4) == 0: - assert miEnabled == False, elineno() - return - - assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" - - mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] - assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" - mi9.append(solution["MatrixInstBM"]) - assert "MIWaveTile" in solution, elineno() + ": missing MIWaveTile" - mi9.extend(solution["MIWaveTile"]) - assert "MIWaveGroup" in solution, elineno() + ": missing MIWaveGroup" - mi9.extend(solution["MIWaveGroup"]) - - assert len(mi4) == 4 and len(mi9) == 9, elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) - - if not miEnabled: - return False - - - wfsize = solution["WavefrontSize"] - waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] - wg0 = mi9[4] * mi9[0] * mi9[7] # Work group 0 - - hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] - hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] - - - miBlock = solution["MIBlock"] - miWaveGroup = solution["MIWaveGroup"] - miWaveTile = solution["MIWaveTile"] - - # Check datatype - if not isSparse: # If it's sparse - if hasMFMA: # and it supports MFMA - if not ( - miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()] - ): # but is invalid MFMA - print1( - f"Looks like {mi4} of type {miDataType.toChar()} is not supported for MFMA {validMFMA[miDataType.toChar()]}" - ) - if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA - assert solution["MFMA_BF16_1K"], elineno() - else: - return not reject( - solution, - printSolutionRejectionReason, - f"Invalid MFMA BFloat16 configuration: {solution}", - ) - elif hasWMMA and (not mi4 in validWMMA): - return not reject( - solution, printSolutionRejectionReason, f"Invalid WMMA configuration: {solution}" - ) - else: - if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): - return not reject( - solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" - ) - - if (not hasMFMA) and hasWMMA: - if isa[0] == 10 or isa[0] == 11: - assert miInputPerThread == mi4[2], elineno() - - # Check MIBlock - assert miBlock[0] == mi4[0], elineno() - assert miBlock[1] == mi4[1], elineno() - assert miBlock[2] == mi4[2], elineno() - assert miBlock[3] == mi4[3], elineno() - assert miBlock[4] == min(wg0 // mi4[0], mi4[3]), elineno() - assert miBlock[5] == mi4[3] // miBlock[4], elineno() - - # Check MIWaveGroup - assert miWaveGroup[0] == min((wg0 // mi4[0]) // miBlock[4], waves), elineno() - assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() - - # Check MIWaveTile - assert miWaveTile[0] == mi9[5], elineno() - assert miWaveTile[1] == mi9[6], elineno() - - # Check MIInputPerThread - miInputPerThread = solution["MIInputPerThread"] - - # If Navi architecture, the input per thread is different - if IsaVersion(10, 0, 0) <= isa <= IsaVersion(11, 0, 2): - assert miInputPerThread == mi4[2], elineno() - else: - assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, f"{elineno()} MIInputPerThread: {miInputPerThread} != {mi4[0]} * {mi4[2]} * {mi4[3]} / {wfsize} = {mi4[0] * mi4[2] * mi4[3] // wfsize}" - - - # miInputPerThreadA = solution["MIInputPerThreadA"] - # miInputPerThreadB = solution["MIInputPerThreadB"] - # miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] - # sparseA = not isSparse if isSparse != 2 else False - # sparseB = isSparse == 2 if isSparse else False - # assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() - # assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() - # assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() - return True diff --git a/tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py deleted file mode 100644 index 4761584d6a..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/ValidWorkGroup.py +++ /dev/null @@ -1,62 +0,0 @@ -################################################################################ -# -# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -################################################################################ - -""" -ValidWorkGroup ---- -Dimensions of the workgroup which will operate on a tile and share lds -Example: ( wg0 x wg1 x LocalSplitU ) -""" - -from typing import Dict - -from Tensile.Common import IsaVersion, IsaInfo - -from .Utilities import elineno - -validWorkGroups = [] -for numThreads in range(32, 1025, 32): - for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: - for sg0 in range(1, numThreads // nsg + 1): - sg1 = numThreads // nsg // sg0 - if sg0 * sg1 * nsg == numThreads: - workGroup = [sg0, sg1, nsg] - validWorkGroups.append(workGroup) - - -def validateWorkGroup(solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: str): - try: - _validateWorkGroup(solution, isaInfoMap) - assert solution["Valid"], f"Solution was rejected: {elineno()}" - return True - except AssertionError as e: - print( - f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" - ) - return False - - -def _validateWorkGroup(solution: dict, isaInfoMap: dict): - assert "WorkGroup" in solution, elineno() - assert solution["WorkGroup"] in validWorkGroups, elineno() diff --git a/tensilelite/Tensile/TensileLogicOLD/__init__.py b/tensilelite/Tensile/TensileLogicOLD/__init__.py deleted file mode 100644 index 46b158ded5..0000000000 --- a/tensilelite/Tensile/TensileLogicOLD/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .Run import main From 0fbd45ae95a21408ed292134c3b18fc390f046b9 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:29:08 +0000 Subject: [PATCH 095/105] fix: only convert 9-item MI in BenchmarkProblems --- tensilelite/Tensile/BenchmarkProblems.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 5ed02b87d4..d9b1b78518 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -108,25 +108,22 @@ def _getCustomKernelSolutionObj( assembler: Assembler, debugConfig: DebugConfig, depthUConfig: DepthUConfig, - isaInfoMap: Dict[str, IsaInfo], + isaInfoMap: Dict[IsaVersion, IsaInfo], directory=CUSTOM_KERNEL_PATH ): """Creates the Solution object for a custom kernel""" sol = getCustomKernelConfig(kernelName, internalSupportParams, directory) mi = sol["MatrixInstruction"] - if len(mi) != 9: - printWarning(f"Custom kernel {kernelName} has MI length {len(mi)}, expected 9.") - isa = next(iter(isaInfoMap.keys())) - wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] workgroup = sol["WorkGroup"] # TODO: this should be deleted once all custom kernel configs MI are length 4. - miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) - sol.update(miParams) + if len(mi) == 9: + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + sol.update(miParams) return Solution( sol, From 4114e28497c0868b4ae41bc45254039214c6db5a Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:36:08 +0000 Subject: [PATCH 096/105] fix: some Tensile Solutions don't have a wavefront size --- tensilelite/Tensile/BenchmarkProblems.py | 2 +- tensilelite/Tensile/SolutionStructs/Utilities.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index d9b1b78518..742fc9b09b 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -116,7 +116,7 @@ def _getCustomKernelSolutionObj( mi = sol["MatrixInstruction"] isa = next(iter(isaInfoMap.keys())) - wavefrontSize = sol["WavefrontSize"] + wavefrontSize = sol.get("WavefrontSize", None) ptype = sol["ProblemType"] workgroup = sol["WorkGroup"] diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index 053b8e85ac..a9a0cc1184 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -24,7 +24,7 @@ import sys import pprint -from typing import Dict +from typing import Dict, Optional from Tensile.Common import IsaVersion, IsaInfo, print1, print2 from Tensile.Common.ValidParameters import validMFMA @@ -67,7 +67,7 @@ def reject(state: dict, printSolutionRejectionReason: bool = True, *args) -> boo def matrixInstructionToMIParameters( mi: list, isa: IsaVersion, - wavefrontSize: int, + wavefrontSize: Optional[int], problemType: dict, workGroup: list, isaInfoMap: Dict[IsaVersion, IsaInfo] @@ -90,7 +90,10 @@ def matrixInstructionToMIParameters( result = {} result["ISA"] = isa - result["WavefrontSize"] = wavefrontSize + + if wavefrontSize: + # Some Solutions used during benchmarking don't have WavefrontSize set on them. + result["WavefrontSize"] = wavefrontSize # Enable F32 XDL math operation only when the input type is f32. enableF32xdl = ( From 4e8c919fef9269b607c9603c0dc30ae963d0407d Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:39:05 +0000 Subject: [PATCH 097/105] fix: some Tensile Solutions don't have a work group --- tensilelite/Tensile/BenchmarkProblems.py | 7 +++++-- tensilelite/Tensile/SolutionStructs/Utilities.py | 13 +++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 742fc9b09b..19769c387e 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -118,14 +118,14 @@ def _getCustomKernelSolutionObj( isa = next(iter(isaInfoMap.keys())) wavefrontSize = sol.get("WavefrontSize", None) ptype = sol["ProblemType"] - workgroup = sol["WorkGroup"] + workgroup = sol.get("WorkGroup", None) # TODO: this should be deleted once all custom kernel configs MI are length 4. if len(mi) == 9: miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) sol.update(miParams) - return Solution( + sol = Solution( sol, debugConfig.printIndexAssignmentInfo, debugConfig.printSolutionRejectionReason, @@ -134,6 +134,9 @@ def _getCustomKernelSolutionObj( assembler, isaInfoMap ) + + print1(f" --DBG-- wavefrontSize: {sol['WavefrontSize']}") + return sol def _generateCustomKernelSolutions( diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index a9a0cc1184..e2760b7364 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -69,7 +69,7 @@ def matrixInstructionToMIParameters( isa: IsaVersion, wavefrontSize: Optional[int], problemType: dict, - workGroup: list, + workGroup: Optional[list], isaInfoMap: Dict[IsaVersion, IsaInfo] ): """ @@ -90,10 +90,6 @@ def matrixInstructionToMIParameters( result = {} result["ISA"] = isa - - if wavefrontSize: - # Some Solutions used during benchmarking don't have WavefrontSize set on them. - result["WavefrontSize"] = wavefrontSize # Enable F32 XDL math operation only when the input type is f32. enableF32xdl = ( @@ -114,7 +110,12 @@ def matrixInstructionToMIParameters( waves = mi[7]* mi[8] wg0 = mi[4] * mi[0] * mi[7] - result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] + if wavefrontSize: + # Some Solutions used during benchmarking don't have WavefrontSize set. + result["WavefrontSize"] = wavefrontSize + if workGroup: + # Some Solutions used during benchmarking don't have WorkGroup set. + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] result["ThreadTile"] = [1, 1] # dummy isSparse = problemType.get("Sparse", 0) From 58ac28d7c238249a03fbc4962c9209419605dceb Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:49:22 +0000 Subject: [PATCH 098/105] fix: add wavefront size to missing custom kernels --- ..._AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s | 1 + ..._UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s | 1 + 2 files changed, 2 insertions(+) diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index ab17bdd17a..cbbd711b93 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -62,6 +62,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index 63bbea1b4c..c4d3be96af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -62,6 +62,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 From a8a636c591398d70f861906fc1c9f5423a09b25c Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:50:33 +0000 Subject: [PATCH 099/105] fix: debug work group --- tensilelite/Tensile/BenchmarkProblems.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 19769c387e..f1fd53f101 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -136,6 +136,7 @@ def _getCustomKernelSolutionObj( ) print1(f" --DBG-- wavefrontSize: {sol['WavefrontSize']}") + print1(f" --DBG-- workGroup: {sol['WorkGroup']}") return sol From d2286f284a1bf5d4fb25d08df4af2db56ab623f4 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:54:52 +0000 Subject: [PATCH 100/105] fix: restore wavefront size and work group --- tensilelite/Tensile/BenchmarkProblems.py | 4 ++-- ...Args_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s | 1 + ...256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s | 1 + tensilelite/Tensile/SolutionStructs/Utilities.py | 12 ++++-------- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index f1fd53f101..9a42305a09 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -116,9 +116,9 @@ def _getCustomKernelSolutionObj( mi = sol["MatrixInstruction"] isa = next(iter(isaInfoMap.keys())) - wavefrontSize = sol.get("WavefrontSize", None) + wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] - workgroup = sol.get("WorkGroup", None) + workgroup = sol["WorkGroup"] # TODO: this should be deleted once all custom kernel configs MI are length 4. if len(mi) == 9: diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index cbbd711b93..f707af8d20 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -63,6 +63,7 @@ custom.config: SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] WavefrontSize: 64 + WorkGroup: [16, 16, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index c4d3be96af..cd7e3373d9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -63,6 +63,7 @@ custom.config: SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] WavefrontSize: 64 + WorkGroup: [16, 16, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index e2760b7364..b722f08f55 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -67,9 +67,9 @@ def reject(state: dict, printSolutionRejectionReason: bool = True, *args) -> boo def matrixInstructionToMIParameters( mi: list, isa: IsaVersion, - wavefrontSize: Optional[int], + wavefrontSize: int, problemType: dict, - workGroup: Optional[list], + workGroup: list, isaInfoMap: Dict[IsaVersion, IsaInfo] ): """ @@ -110,12 +110,8 @@ def matrixInstructionToMIParameters( waves = mi[7]* mi[8] wg0 = mi[4] * mi[0] * mi[7] - if wavefrontSize: - # Some Solutions used during benchmarking don't have WavefrontSize set. - result["WavefrontSize"] = wavefrontSize - if workGroup: - # Some Solutions used during benchmarking don't have WorkGroup set. - result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] + result["WavefrontSize"] = wavefrontSize + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] result["ThreadTile"] = [1, 1] # dummy isSparse = problemType.get("Sparse", 0) From 16e9c5bbfc72badeca8eae171915c3039dc7376d Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 20:59:34 +0000 Subject: [PATCH 101/105] Revert "fix: debug work group" This reverts commit a8a636c591398d70f861906fc1c9f5423a09b25c. --- tensilelite/Tensile/BenchmarkProblems.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 9a42305a09..6d4b303c30 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -136,7 +136,6 @@ def _getCustomKernelSolutionObj( ) print1(f" --DBG-- wavefrontSize: {sol['WavefrontSize']}") - print1(f" --DBG-- workGroup: {sol['WorkGroup']}") return sol From 0e827dd7dba3db939f8620febcd103c0e3c5505c Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 21:01:36 +0000 Subject: [PATCH 102/105] fix: make work group optional but not wavefront sz --- tensilelite/Tensile/BenchmarkProblems.py | 5 +++-- tensilelite/Tensile/SolutionStructs/Utilities.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 6d4b303c30..f1fd53f101 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -116,9 +116,9 @@ def _getCustomKernelSolutionObj( mi = sol["MatrixInstruction"] isa = next(iter(isaInfoMap.keys())) - wavefrontSize = sol["WavefrontSize"] + wavefrontSize = sol.get("WavefrontSize", None) ptype = sol["ProblemType"] - workgroup = sol["WorkGroup"] + workgroup = sol.get("WorkGroup", None) # TODO: this should be deleted once all custom kernel configs MI are length 4. if len(mi) == 9: @@ -136,6 +136,7 @@ def _getCustomKernelSolutionObj( ) print1(f" --DBG-- wavefrontSize: {sol['WavefrontSize']}") + print1(f" --DBG-- workGroup: {sol['WorkGroup']}") return sol diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py index b722f08f55..659a2a83a2 100644 --- a/tensilelite/Tensile/SolutionStructs/Utilities.py +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -69,7 +69,7 @@ def matrixInstructionToMIParameters( isa: IsaVersion, wavefrontSize: int, problemType: dict, - workGroup: list, + workGroup: Optional[list], isaInfoMap: Dict[IsaVersion, IsaInfo] ): """ @@ -111,7 +111,9 @@ def matrixInstructionToMIParameters( wg0 = mi[4] * mi[0] * mi[7] result["WavefrontSize"] = wavefrontSize - result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] + if workGroup: + # Some Solutions used during benchmarking don't have WorkGroup set. + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] result["ThreadTile"] = [1, 1] # dummy isSparse = problemType.get("Sparse", 0) From 2a75b57f6804057a3c9be30399b5a74ce11c71a4 Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 21:06:21 +0000 Subject: [PATCH 103/105] fix: add wavefront size to missing custom kernels --- tensilelite/Tensile/BenchmarkProblems.py | 2 +- ...AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s | 1 - ...UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s | 1 - ...V_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s | 1 + ...serArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s | 1 + ...V_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s | 1 + ...serArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s | 1 + 7 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index f1fd53f101..40413b8cb0 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -116,7 +116,7 @@ def _getCustomKernelSolutionObj( mi = sol["MatrixInstruction"] isa = next(iter(isaInfoMap.keys())) - wavefrontSize = sol.get("WavefrontSize", None) + wavefrontSize = sol["WavefrontSize"] ptype = sol["ProblemType"] workgroup = sol.get("WorkGroup", None) diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index f707af8d20..cbbd711b93 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -63,7 +63,6 @@ custom.config: SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] WavefrontSize: 64 - WorkGroup: [16, 16, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index cd7e3373d9..c4d3be96af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -63,7 +63,6 @@ custom.config: SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] WavefrontSize: 64 - WorkGroup: [16, 16, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index 566eb28e47..ebb87bef67 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index cfc8b39984..225e125618 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s index 9732eedb6c..862f85074c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 16, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s index f4f20252a4..b156ac1179 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 16, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 From 9635690833d06cb19ad128cb543234969c5fcc3d Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 21:11:49 +0000 Subject: [PATCH 104/105] BOOKMARK: delete MatrrixInstructions from validParameters --- tensilelite/Tensile/Common/ValidParameters.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index fecbf6b49f..8006df0528 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -516,29 +516,6 @@ "ThreadTile": validThreadTiles, "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 "WavefrontSize": [32, 64], - # MatrixInstruction: (M x N x K x B) - # XDLOPS tile definition, only valid for gfx908, gfx90a - # MxNxKxB specifies matrix instruction variants - # MxNxB determines the shape of the C tile each instruction worked on - # K determines the unroll depth - # If empty, do not use these instructions - # - # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) - # (Note: MxN means M-by-N in the following comments) - # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: - # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed - # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed - # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels - # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) - # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads - # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 - # Putting it all together: - # - [32, 32, 1, 2, 1, 4, 1, 2, 2] - # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ - # MatrixInst BlkM WT Wave - # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile - # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used - "MatrixInstruction": validMatrixInstructions, # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. # MI output data between each threads are along N dims. # But global memory is along M dim continuous. From 266e9b81bc2bc7b07d94696447d284e4f8d2fa3a Mon Sep 17 00:00:00 2001 From: Braden Stefanuk <121893577+bstefanuk@users.noreply.github.com> Date: Mon, 3 Mar 2025 21:21:28 +0000 Subject: [PATCH 105/105] just set MatrixInstruction to -1 instead --- tensilelite/Tensile/Common/ValidParameters.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py index 8006df0528..224a9cfe5f 100644 --- a/tensilelite/Tensile/Common/ValidParameters.py +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -516,6 +516,31 @@ "ThreadTile": validThreadTiles, "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 "WavefrontSize": [32, 64], + # MatrixInstruction: (M x N x K x B) + # XDLOPS tile definition, only valid for gfx908, gfx90a + # MxNxKxB specifies matrix instruction variants + # MxNxB determines the shape of the C tile each instruction worked on + # K determines the unroll depth + # If empty, do not use these instructions + # + # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + # (Note: MxN means M-by-N in the following comments) + # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + # Putting it all together: + # - [32, 32, 1, 2, 1, 4, 1, 2, 2] + # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + # MatrixInst BlkM WT Wave + # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + # NOTE: MatrixInstruction is no longer validated through this structure, but is instead validated via the + # ``TensileLogic`` program. + "MatrixInstruction": -1, # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. # MI output data between each threads are along N dims. # But global memory is along M dim continuous.